{"id":31482286,"url":"https://github.com/snowflake-labs/schemachange","last_synced_at":"2025-10-02T07:45:16.177Z","repository":{"id":37936579,"uuid":"146255484","full_name":"Snowflake-Labs/schemachange","owner":"Snowflake-Labs","description":"A Database Change Management tool for Snowflake","archived":false,"fork":false,"pushed_at":"2025-09-05T18:34:41.000Z","size":1392,"stargazers_count":588,"open_issues_count":92,"forks_count":263,"subscribers_count":28,"default_branch":"master","last_synced_at":"2025-09-29T13:02:23.654Z","etag":null,"topics":[],"latest_commit_sha":null,"homepage":"","language":"Python","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"apache-2.0","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/Snowflake-Labs.png","metadata":{"files":{"readme":"README.md","changelog":"CHANGELOG.md","contributing":".github/CONTRIBUTING.md","funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null,"zenodo":null,"notice":"NOTICE","maintainers":null,"copyright":null,"agents":null,"dco":null,"cla":null}},"created_at":"2018-08-27T06:24:28.000Z","updated_at":"2025-09-27T04:48:31.000Z","dependencies_parsed_at":"2024-05-13T17:40:20.570Z","dependency_job_id":"e5d43e27-66b2-4766-9400-2d639cdc1741","html_url":"https://github.com/Snowflake-Labs/schemachange","commit_stats":{"total_commits":149,"total_committers":22,"mean_commits":"6.7727272727272725","dds":0.8187919463087249,"last_synced_commit":"f5dad1ac3d93e82c305dcb1a58eecd599028e053"},"previous_names":[],"tags_count":7,"template":false,"template_full_name":null,"purl":"pkg:github/Snowflake-Labs/schemachange","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/Snowflake-Labs%2Fschemachange","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/Snowflake-Labs%2Fschemachange/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/Snowflake-Labs%2Fschemachange/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/Snowflake-Labs%2Fschemachange/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/Snowflake-Labs","download_url":"https://codeload.github.com/Snowflake-Labs/schemachange/tar.gz/refs/heads/master","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/Snowflake-Labs%2Fschemachange/sbom","scorecard":{"id":131597,"data":{"date":"2025-08-04","repo":{"name":"github.com/Snowflake-Labs/schemachange","commit":"abfb4f34fe2897c8eda8c08751f335dfdfa2f52f"},"scorecard":{"version":"v5.2.1-28-gc1d103a9","commit":"c1d103a9bb9f635ec7260bf9aa0699466fa4be0e"},"score":5.2,"checks":[{"name":"Maintained","score":10,"reason":"18 commit(s) and 4 issue activity found in the last 90 days -- score normalized to 10","details":null,"documentation":{"short":"Determines if the project is \"actively maintained\".","url":"https://github.com/ossf/scorecard/blob/c1d103a9bb9f635ec7260bf9aa0699466fa4be0e/docs/checks.md#maintained"}},{"name":"Code-Review","score":8,"reason":"Found 13/15 approved changesets -- score normalized to 8","details":null,"documentation":{"short":"Determines if the project requires human code review before pull requests (aka merge requests) are merged.","url":"https://github.com/ossf/scorecard/blob/c1d103a9bb9f635ec7260bf9aa0699466fa4be0e/docs/checks.md#code-review"}},{"name":"Dangerous-Workflow","score":10,"reason":"no dangerous workflow patterns detected","details":null,"documentation":{"short":"Determines if the project's GitHub Action workflows avoid dangerous patterns.","url":"https://github.com/ossf/scorecard/blob/c1d103a9bb9f635ec7260bf9aa0699466fa4be0e/docs/checks.md#dangerous-workflow"}},{"name":"Binary-Artifacts","score":10,"reason":"no binaries found in the repo","details":null,"documentation":{"short":"Determines if the project has generated executable (binary) artifacts in the source repository.","url":"https://github.com/ossf/scorecard/blob/c1d103a9bb9f635ec7260bf9aa0699466fa4be0e/docs/checks.md#binary-artifacts"}},{"name":"CII-Best-Practices","score":0,"reason":"no effort to earn an OpenSSF best practices badge detected","details":null,"documentation":{"short":"Determines if the project has an OpenSSF (formerly CII) Best Practices Badge.","url":"https://github.com/ossf/scorecard/blob/c1d103a9bb9f635ec7260bf9aa0699466fa4be0e/docs/checks.md#cii-best-practices"}},{"name":"Token-Permissions","score":0,"reason":"detected GitHub workflow tokens with excessive permissions","details":["Info: topLevel 'contents' permission set to 'read': .github/workflows/dependency-review.yml:5","Warn: no topLevel permission defined: .github/workflows/dev-pytest.yml:1","Warn: no topLevel permission defined: .github/workflows/master-pytest.yml:1","Info: topLevel 'contents' permission set to 'read': .github/workflows/python-publish.yml:16","Info: no jobLevel write permissions found"],"documentation":{"short":"Determines if the project's workflows follow the principle of least privilege.","url":"https://github.com/ossf/scorecard/blob/c1d103a9bb9f635ec7260bf9aa0699466fa4be0e/docs/checks.md#token-permissions"}},{"name":"License","score":10,"reason":"license file detected","details":["Info: project has a license file: LICENSE:0","Info: FSF or OSI recognized license: Apache License 2.0: LICENSE:0"],"documentation":{"short":"Determines if the project has defined a license.","url":"https://github.com/ossf/scorecard/blob/c1d103a9bb9f635ec7260bf9aa0699466fa4be0e/docs/checks.md#license"}},{"name":"Fuzzing","score":0,"reason":"project is not fuzzed","details":["Warn: no fuzzer integrations found"],"documentation":{"short":"Determines if the project uses fuzzing.","url":"https://github.com/ossf/scorecard/blob/c1d103a9bb9f635ec7260bf9aa0699466fa4be0e/docs/checks.md#fuzzing"}},{"name":"Signed-Releases","score":-1,"reason":"no releases found","details":null,"documentation":{"short":"Determines if the project cryptographically signs release artifacts.","url":"https://github.com/ossf/scorecard/blob/c1d103a9bb9f635ec7260bf9aa0699466fa4be0e/docs/checks.md#signed-releases"}},{"name":"Branch-Protection","score":-1,"reason":"internal error: error during branchesHandler.setup: internal error: githubv4.Query: Resource not accessible by integration","details":null,"documentation":{"short":"Determines if the default and release branches are protected with GitHub's branch protection settings.","url":"https://github.com/ossf/scorecard/blob/c1d103a9bb9f635ec7260bf9aa0699466fa4be0e/docs/checks.md#branch-protection"}},{"name":"Packaging","score":10,"reason":"packaging workflow detected","details":["Info: Project packages its releases by way of GitHub Actions.: .github/workflows/python-publish.yml:19"],"documentation":{"short":"Determines if the project is published as a package that others can easily download, install, easily update, and uninstall.","url":"https://github.com/ossf/scorecard/blob/c1d103a9bb9f635ec7260bf9aa0699466fa4be0e/docs/checks.md#packaging"}},{"name":"Security-Policy","score":0,"reason":"security policy file not detected","details":["Warn: no security policy file detected","Warn: no security file to analyze","Warn: no security file to analyze","Warn: no security file to analyze"],"documentation":{"short":"Determines if the project has published a security policy.","url":"https://github.com/ossf/scorecard/blob/c1d103a9bb9f635ec7260bf9aa0699466fa4be0e/docs/checks.md#security-policy"}},{"name":"Pinned-Dependencies","score":0,"reason":"dependency not pinned by hash detected -- score normalized to 0","details":["Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/dependency-review.yml:12: update your workflow using https://app.stepsecurity.io/secureworkflow/Snowflake-Labs/schemachange/dependency-review.yml/master?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/dependency-review.yml:14: update your workflow using https://app.stepsecurity.io/secureworkflow/Snowflake-Labs/schemachange/dependency-review.yml/master?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/dev-pytest.yml:37: update your workflow using https://app.stepsecurity.io/secureworkflow/Snowflake-Labs/schemachange/dev-pytest.yml/master?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/dev-pytest.yml:39: update your workflow using https://app.stepsecurity.io/secureworkflow/Snowflake-Labs/schemachange/dev-pytest.yml/master?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/master-pytest.yml:41: update your workflow using https://app.stepsecurity.io/secureworkflow/Snowflake-Labs/schemachange/master-pytest.yml/master?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/master-pytest.yml:43: update your workflow using https://app.stepsecurity.io/secureworkflow/Snowflake-Labs/schemachange/master-pytest.yml/master?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/python-publish.yml:24: update your workflow using https://app.stepsecurity.io/secureworkflow/Snowflake-Labs/schemachange/python-publish.yml/master?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/python-publish.yml:26: update your workflow using https://app.stepsecurity.io/secureworkflow/Snowflake-Labs/schemachange/python-publish.yml/master?enable=pin","Warn: containerImage not pinned by hash: Dockerfile:1: pin your Docker image by updating python:3.9 to python:3.9@sha256:754dbbaf5fe730bb2460efb3300293c62c222f74fbf8534ed23691c617c9609b","Warn: containerImage not pinned by hash: Dockerfile-src:1: pin your Docker image by updating python:3.9 to python:3.9@sha256:754dbbaf5fe730bb2460efb3300293c62c222f74fbf8534ed23691c617c9609b","Warn: containerImage not pinned by hash: Dockerfile-src:13: pin your Docker image by updating python:3.9 to python:3.9@sha256:754dbbaf5fe730bb2460efb3300293c62c222f74fbf8534ed23691c617c9609b","Warn: pipCommand not pinned by hash: Dockerfile:3","Warn: pipCommand not pinned by hash: Dockerfile-src:8","Warn: pipCommand not pinned by hash: Dockerfile-src:9","Warn: pipCommand not pinned by hash: .github/workflows/dev-pytest.yml:47","Warn: pipCommand not pinned by hash: .github/workflows/dev-pytest.yml:48","Warn: pipCommand not pinned by hash: .github/workflows/dev-pytest.yml:49","Warn: pipCommand not pinned by hash: .github/workflows/dev-pytest.yml:50","Warn: pipCommand not pinned by hash: .github/workflows/master-pytest.yml:51","Warn: pipCommand not pinned by hash: .github/workflows/master-pytest.yml:52","Warn: pipCommand not pinned by hash: .github/workflows/master-pytest.yml:53","Warn: pipCommand not pinned by hash: .github/workflows/master-pytest.yml:54","Warn: pipCommand not pinned by hash: .github/workflows/python-publish.yml:31","Warn: pipCommand not pinned by hash: .github/workflows/python-publish.yml:32","Info:   0 out of   8 GitHub-owned GitHubAction dependencies pinned","Info:   1 out of   1 third-party GitHubAction dependencies pinned","Info:   0 out of   3 containerImage dependencies pinned","Info:   1 out of  14 pipCommand dependencies pinned"],"documentation":{"short":"Determines if the project has declared and pinned the dependencies of its build process.","url":"https://github.com/ossf/scorecard/blob/c1d103a9bb9f635ec7260bf9aa0699466fa4be0e/docs/checks.md#pinned-dependencies"}},{"name":"Vulnerabilities","score":0,"reason":"10 existing vulnerabilities detected","details":["Warn: Project is vulnerable to: GHSA-cpwx-vrp4-4pq7","Warn: Project is vulnerable to: GHSA-gmj6-6f8f-6699","Warn: Project is vulnerable to: GHSA-h5c8-rqwp-cp95","Warn: Project is vulnerable to: GHSA-h75v-3vvj-5mfj","Warn: Project is vulnerable to: GHSA-q2x7-8rv6-6q7h","Warn: Project is vulnerable to: GHSA-4r6j-fwcx-94cf","Warn: Project is vulnerable to: PYSEC-2024-191 / GHSA-5vvg-pvhp-hv2m","Warn: Project is vulnerable to: PYSEC-2023-88 / GHSA-5w5m-pfw9-c8fp","Warn: Project is vulnerable to: PYSEC-2025-27","Warn: Project is vulnerable to: PYSEC-2025-28"],"documentation":{"short":"Determines if the project has open, known unfixed vulnerabilities.","url":"https://github.com/ossf/scorecard/blob/c1d103a9bb9f635ec7260bf9aa0699466fa4be0e/docs/checks.md#vulnerabilities"}},{"name":"SAST","score":3,"reason":"SAST tool is not run on all commits -- score normalized to 3","details":["Warn: 11 commits out of 30 are checked with a SAST tool"],"documentation":{"short":"Determines if the project uses static code analysis.","url":"https://github.com/ossf/scorecard/blob/c1d103a9bb9f635ec7260bf9aa0699466fa4be0e/docs/checks.md#sast"}}]},"last_synced_at":"2025-08-16T05:21:14.986Z","repository_id":37936579,"created_at":"2025-08-16T05:21:14.986Z","updated_at":"2025-08-16T05:21:14.986Z"},"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":277974426,"owners_count":25908396,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","status":"online","status_checked_at":"2025-10-02T02:00:08.890Z","response_time":67,"last_error":null,"robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":true,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":[],"created_at":"2025-10-02T07:45:14.819Z","updated_at":"2025-10-02T07:45:16.165Z","avatar_url":"https://github.com/Snowflake-Labs.png","language":"Python","funding_links":[],"categories":[],"sub_categories":[],"readme":"# schemachange\n\n\u003cimg src=\"https://github.com/user-attachments/assets/8bc170c9-4171-48c7-812c-6d76c07ee364\" alt=\"schemachange\" title=\"schemachange logo\" width=\"600\" /\u003e\n\n*Looking for snowchange? You've found the right spot. snowchange has been renamed to schemachange.*\n\n[![pytest](https://github.com/Snowflake-Labs/schemachange/actions/workflows/master-pytest.yml/badge.svg)](https://github.com/Snowflake-Labs/schemachange/actions/workflows/master-pytest.yml)\n[![PyPI](https://img.shields.io/pypi/v/schemachange.svg)](https://pypi.org/project/schemachange)\n\n## Overview\n\nschemachange is a simple python based tool to manage all of your [Snowflake](https://www.snowflake.com/) objects. It\nfollows an Imperative-style approach to Database Change Management (DCM) and was inspired by\nthe [Flyway database migration tool](https://www.red-gate.com/products/flyway/community/). When combined with a version control system and a CI/CD\ntool, database changes can be approved and deployed through a pipeline using modern software delivery practices. As such\nschemachange plays a critical role in enabling Database (or Data) DevOps.\n\nDCM tools (also known as Database Migration, Schema Change Management, or Schema Migration tools) follow one of two\napproaches: Declarative or Imperative. For a background on Database DevOps, including a discussion on the differences\nbetween the Declarative and Imperative approaches, please read\nthe [Embracing Agile Software Delivery and DevOps with Snowflake](https://www.snowflake.com/blog/embracing-agile-software-delivery-and-devops-with-snowflake/)\nblog post.\n\nFor the complete list of changes made to schemachange check out the [CHANGELOG](CHANGELOG.md).\n\nTo learn more about making a contribution to schemachange, please see our [Contributing guide](.github/CONTRIBUTING.md).\n\n**Please note** that schemachange is a community-developed tool, not an official Snowflake offering. It comes with no\nsupport or warranty.\n\n## Table of Contents\n\n1. [Overview](#overview)\n1. [Project Structure](#project-structure)\n    1. [Folder Structure](#folder-structure)\n1. [Change Scripts](#change-scripts)\n    1. [Versioned Script Naming](#versioned-script-naming)\n    1. [Repeatable Script Naming](#repeatable-script-naming)\n    1. [Always Script Naming](#always-script-naming)\n    1. [Script Requirements](#script-requirements)\n    1. [Using Variables in Scripts](#using-variables-in-scripts)\n        1. [Secrets filtering](#secrets-filtering)\n    1. [Jinja templating engine](#jinja-templating-engine)\n    1. [Gotchas](#gotchas)\n1. [Change History Table](#change-history-table)\n1. [Authentication](#authentication)\n    1. [Password Authentication](#password-authentication)\n    1. [External OAuth Authentication](#external-oauth-authentication)\n    1. [External Browser Authentication](#external-browser-authentication)\n    1. [Okta Authentication](#okta-authentication)\n    1. [Private Key Authentication](#private-key-authentication)\n1. [Configuration](#configuration)\n    1. [YAML Config File](#yaml-config-file)\n        1. [Yaml Jinja support](#yaml-jinja-support)\n    1. [connections.toml File](#connectionstoml-file)\n1. [Commands](#commands)\n    1. [deploy](#deploy)\n    1. [render](#render)\n1. [Running schemachange](#running-schemachange)\n    1. [Prerequisites](#prerequisites)\n    1. [Running the Script](#running-the-script)\n1. [Integrating With DevOps](#integrating-with-devops)\n    1. [Sample DevOps Process Flow](#sample-devops-process-flow)\n    1. [Using in a CI/CD Pipeline](#using-in-a-cicd-pipeline)\n1. [Maintainers](#maintainers)\n1. [Third Party Packages](#third-party-packages)\n1. [Legal](#legal)\n\n## Project Structure\n\n### Folder Structure\n\nschemachange expects a directory structure like the following to exist:\n\n```\n(project_root)\n|\n|-- folder_1\n    |-- V1.1.1__first_change.sql\n    |-- V1.1.2__second_change.sql\n    |-- R__sp_add_sales.sql\n    |-- R__fn_get_timezone.sql\n|-- folder_2\n    |-- folder_3\n        |-- V1.1.3__third_change.sql\n        |-- R__fn_sort_ascii.sql\n```\n\nThe schemachange folder structure is very flexible. The `project_root` folder is specified with the `-f`\nor `--root-folder` argument. schemachange only pays attention to the filenames, not the paths. Therefore, under\nthe `project_root` folder you are free to arrange the change scripts any way you see fit. You can have as many\nsubfolders (and nested subfolders) as you would like.\n\n## Change Scripts\n\n### Versioned Script Naming\n\nVersioned change scripts follow a similar naming convention to that used\nby [Flyway Versioned Migrations](https://documentation.red-gate.com/fd/versioned-migrations-273973333.html). The script name\nmust follow this pattern (image taken\nfrom [Flyway docs](https://documentation.red-gate.com/fd/versioned-migrations-273973333.html)):\n\n\u003cimg src=\"https://github.com/user-attachments/assets/a71297d9-4a3c-4d30-82d3-c634be88fe54\" alt=\"Flyway naming conventions\" title=\"Flyway naming conventions\" width=\"300\" /\u003e\n\nWith the following rules for each part of the filename:\n\n* **Prefix**: The letter 'V' for versioned change\n* **Version**: A unique version number with dots or underscores separating as many number parts as you like\n* **Separator**: __ (two underscores)\n* **Description**: An arbitrary description with words separated by underscores or spaces (can not include two\n  underscores)\n* **Suffix**: .sql or .sql.jinja\n\nFor example, a script name that follows this convention is: `V1.1.1__first_change.sql`. As with Flyway, the unique\nversion string is very flexible. You just need to be consistent and always use the same convention, like 3 sets of\nnumbers separated by periods. Here are a few valid version strings:\n\n* 1.1\n* 1_1\n* 1.2.3\n* 1_2_3\n\nEvery script within a database folder must have a unique version number. schemachange will check for duplicate version\nnumbers and throw an error if it finds any. This helps to ensure that developers who are working in parallel don't\naccidentally (re-)use the same version number.\n\n### Repeatable Script Naming\n\nRepeatable change scripts follow a similar naming convention to that used\nby [Flyway Versioned Migrations](https://documentation.red-gate.com/fd/repeatable-migrations-273973335.html). The\nscript name must follow this pattern (image taken\nfrom [Flyway docs](https://documentation.red-gate.com/fd/repeatable-migrations-273973335.html):\n\n\u003cimg src=\"https://github.com/user-attachments/assets/06abd883-58b7-42d5-97b2-581158d8b121\" alt=\"Flyway naming conventions\" title=\"Flyway naming conventions\" width=\"300\" /\u003e\n\ne.g:\n\n* R__sp_add_sales.sql\n* R__fn_get_timezone.sql\n* R__fn_sort_ascii.sql\n\nAll repeatable change scripts are applied each time the utility is run, if there is a change in the file.\nRepeatable scripts could be used for maintaining code that always needs to be applied in its entirety. e.g. stores\nprocedures, functions and view definitions etc.\n\nJust like Flyway, within a single migration run, repeatable scripts are always applied after all pending versioned\nscripts have been executed. Repeatable scripts are applied in alphabetical order of their description.\n\n### Always Script Naming\n\nAlways change scripts are executed with every run of schemachange. This is an addition to the implementation\nof [Flyway Versioned Migrations](https://documentation.red-gate.com/fd/versioned-migrations-273973333.html).\nThe script name must follow this pattern:\n\n`A__Some_description.sql`\n\ne.g.\n\n* A__add_user.sql\n* A__assign_roles.sql\n\nThis type of change script is useful for an environment set up after cloning. Always scripts are applied always last.\n\n### Script Requirements\n\nschemachange is designed to be very lightweight and not impose too many limitations. Each change script can have any\nnumber of SQL statements within it and must supply the necessary context, like database and schema names. The context\ncan be supplied by using an explicit `USE \u003cDATABASE\u003e` command or by naming all objects with a three-part\nname (`\u003cdatabase name\u003e.\u003cschema name\u003e.\u003cobject name\u003e`). schemachange will simply run the contents of each script against\nthe target Snowflake account, in the correct order. After each script, Schemachange will execute \"reset\" the context (\nrole, warehouse, database, schema) to the values used to configure the connector.\n\n### Using Variables in Scripts\n\nschemachange supports the jinja engine for a variable replacement strategy. One important use of variables is to support\nmultiple environments (dev, test, prod) in a single Snowflake account by dynamically changing the database name during\ndeployment. To use a variable in a change script, use this syntax anywhere in the script: `{{ variable1 }}`.\n\nTo pass variables to schemachange, check out the [Configuration](#configuration) section below. You can either use\nthe `--vars` command line parameter or the YAML config file `schemachange-config.yml`. For the command line version you\ncan pass variables like this: `--vars '{\"variable1\": \"value\", \"variable2\": \"value2\"}'`. This parameter accepts a flat\nJSON object formatted as a string.\n\n\u003e *Nested objects and arrays don't make sense at this point and aren't supported.*\n\nschemachange will replace any variable placeholders before running your change script code and will throw an error if it\nfinds any variable placeholders that haven't been replaced.\n\n#### Secrets filtering\n\nWhile many CI/CD tools already have the capability to filter secrets, it is best that any tool also does not output\nsecrets to the console or logs. Schemachange implements secrets filtering in a number of areas to ensure secrets are not\nwriten to the console or logs. The only exception is the `render` command which will display secrets.\n\nA secret is just a standard variable that has been tagged as a secret. This is determined using a naming convention and\neither of the following will tag a variable as a secret:\n\n1. The variable name has the word `secret` in it.\n   ```yaml\n      config-version: 1\n      vars:\n         bucket_name: S3://......  # not a secret\n         secret_key: 567576D8E  # a secret\n   ```\n2. The variable is a child of a key named `secrets`.\n   ```yaml\n      config-version: 1\n      vars:\n      secrets:\n         my_key: 567576D8E # a secret\n      aws:\n         bucket_name: S3://......  # not a secret\n         secrets:\n            encryption_key: FGDSUUEHDHJK # a secret\n            us_east_1:\n               encryption_key: sdsdsd # a secret\n   ```\n\n### Jinja templating engine\n\nschemachange uses the Jinja templating engine internally and\nsupports: [expressions](https://jinja.palletsprojects.com/en/3.0.x/templates/#expressions), [macros](https://jinja.palletsprojects.com/en/3.0.x/templates/#macros), [includes](https://jinja.palletsprojects.com/en/3.0.x/templates/#include)\nand [template inheritance](https://jinja.palletsprojects.com/en/3.0.x/templates/#template-inheritance).\n\nThese files can be stored in the root-folder but schemachange also provides a separate modules\nfolder `--modules-folder`. This allows common logic to be stored outside of the main changes scripts.\nThe [demo/citibike_demo_jinja](demo/citibike_demo_jinja) has a simple example that demonstrates this.\n\nschemachange uses Jinja's [`PrefixLoader`](https://jinja.palletsprojects.com/en/stable/api/#jinja2.PrefixLoader), so\nregardless of the `--modules-folder` that's used, the file paths (such as those passed to [`include`](https://jinja.palletsprojects.com/en/stable/templates/#include))\nshould be prefixed with `modules/`.\n\nThe Jinja auto-escaping feature is disabled in schemachange, this feature in Jinja is currently designed for where the\noutput language is HTML/XML. So if you are using schemachange with untrusted inputs you will need to handle this within\nyour change scripts.\n\n### Gotchas\n\nWithin change scripts:\n\n- [Snowflake Scripting blocks need delimiters](https://docs.snowflake.com/en/developer-guide/snowflake-scripting/running-examples#introduction)\n- [The last line can't be a comment](https://github.com/Snowflake-Labs/schemachange/issues/130)\n\n## Change History Table\n\nschemachange records all applied changes scripts to the change history table. By default, schemachange will attempt to\nlog all activities to the `METADATA.SCHEMACHANGE.CHANGE_HISTORY` table. The name and location of the change history\ntable can be overriden via a command line argument (`-c` or `--change-history-table`) or the `schemachange-config.yml`\nfile ( `change-history-table`). The value passed to the parameter can have a one, two, or three part name (e.g. \"\nTABLE_NAME\", or \"SCHEMA_NAME.TABLE_NAME\", or \" DATABASE_NAME.SCHEMA_NAME.TABLE_NAME\"). This can be used to support\nmultiple environments (dev, test, prod) or multiple subject areas within the same Snowflake account.\n\nBy default, schemachange will not try to create the change history table, and it will fail if the table does not exist.\nThis behavior can be altered by passing in the `--create-change-history-table` argument or adding\n`create-change-history-table: true` to the `schemachange-config.yml` file. Even with the `--create-change-history-table`\nparameter, schemachange will not attempt to create the database for the change history table. That must be created\nbefore running schemachange.\n\nThe structure of the `CHANGE_HISTORY` table is as follows:\n\n| Column Name    | Type          | Example                       |\n|----------------|---------------|-------------------------------|\n| VERSION        | VARCHAR       | 1.1.1                         |\n| DESCRIPTION    | VARCHAR       | First change                  |\n| SCRIPT         | VARCHAR       | V1.1.1__first_change.sql      |\n| SCRIPT_TYPE    | VARCHAR       | V                             |\n| CHECKSUM       | VARCHAR       | 38e5ba03b1a6d2...             |\n| EXECUTION_TIME | NUMBER        | 4                             |\n| STATUS         | VARCHAR       | Success                       |\n| INSTALLED_BY   | VARCHAR       | SNOWFLAKE_USER                |\n| INSTALLED_ON   | TIMESTAMP_LTZ | 2020-03-17 12:54:33.056 -0700 |\n\nA new row will be added to this table every time a change script has been applied to the database. schemachange will use\nthis table to identify which changes have been applied to the database and will not apply the same version more than\nonce.\n\nHere is the current schema DDL for the change history table (found in the [schemachange/cli.py](schemachange/cli.py)\nscript), in case you choose to create it manually and not use the `--create-change-history-table` parameter:\n\n```sql\nCREATE TABLE IF NOT EXISTS SCHEMACHANGE.CHANGE_HISTORY\n(\n    VERSION        VARCHAR,\n    DESCRIPTION    VARCHAR,\n    SCRIPT         VARCHAR,\n    SCRIPT_TYPE    VARCHAR,\n    CHECKSUM       VARCHAR,\n    EXECUTION_TIME NUMBER,\n    STATUS         VARCHAR,\n    INSTALLED_BY   VARCHAR,\n    INSTALLED_ON   TIMESTAMP_LTZ\n)\n```\n\n## Authentication\n\nSchemachange supports the many of the authentication methods supported by\nthe [Snowflake Python Connector](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-connect).\nThe authenticator can be set by setting an `authenticator` in the [connections.toml](#connectionstoml-file) file\n\nThe following authenticators are supported:\n\n- `snowflake`: [Password](#password-authentication)\n- `oauth`: [External OAuth](#external-oauth-authentication)\n- `externalbrowser`: [Browser-based SSO](#external-browser-authentication)\n- `https://\u003cokta_account_name\u003e.okta.com`: [Okta SSO](#okta-authentication)\n- `snowflake_jwt`: [Private Key](#private-key-authentication)\n\nIf an authenticator is unsupported, an exception will be raised.\n\n### Password Authentication\n\nPassword authentication is the default authenticator. Supplying `snowflake` as your authenticator will set it\nexplicitly. A `password` must be supplied in the [connections.toml](#connectionstoml-file) file\n\n### External OAuth Authentication\n\nExternal OAuth authentication can be selected by supplying `oauth` as your authenticator. A `token_file_path` must be\nsupplied in the [connections.toml](#connectionstoml-file) file\n\n**Schemachange no longer supports the `--oauth-config` option.**  Prior to the 4.0 release, this library supported\nsupplying an `--oauth-config` that would be used to fetch an OAuth token via the `requests` library. This required\nSchemachange to keep track of connection arguments that could otherwise be passed directly to the Snowflake Python\nconnector. Maintaining this logic in Schemachange added unnecessary complication to the repo and prevented access to\nrecent connector parameterization features offered by the Snowflake connector.\n\n### External Browser Authentication\n\nExternal browser authentication can be selected by supplying `externalbrowser` as your authenticator. The client will be\nprompted to authenticate in a browser that pops up. Refer to\nthe [documentation](https://docs.snowflake.com/en/user-guide/admin-security-fed-auth-use.html#setting-up-browser-based-sso)\nto cache the token to minimize the number of times the browser pops up to authenticate the user.\n\n### Okta Authentication\n\nExternal browser authentication can be selected by supplying your Okta endpoint as your authenticator (e.g.\n`https://\u003corg_name\u003e.okta.com`). For clients that do not have a browser, can use the popular SaaS Idp option to connect\nvia Okta. A `password` must be supplied in the [connections.toml](#connectionstoml-file) file\n\n_** NOTE**: Please disable Okta MFA for the user who uses Native SSO authentication with client drivers. Please consult\nyour Okta administrator for more information._\n\n### Private Key Authentication\n\nPrivate key authentication can be selected by supplying `snowflake_jwt` as your authenticator. The filepath to a\nSnowflake user-encrypted private key must be supplied as `private_key_file` in the [connections.toml](#connectionstoml-file)\nfile. If the private key file is password protected, supply the password as `private_key_file_pwd` in\nthe [connections.toml](#connectionstoml-file) file. If the variable is not set, the Snowflake Python connector will\nassume the private key is not encrypted.\n\n## Configuration\n\nAs of version 4.0, Snowflake connection parameters must be supplied via\na [connections.toml file](#connectionstoml-file). Command-line and yaml arguments will still be supported with a\ndeprecation warning until support is completely dropped.\n\nSchemachange-specific parameters can be supplied in two different ways (in order of priority):\n\n1. Command Line Arguments\n2. YAML config file\n\n**Note:** As of 4.0, `vars` provided via command-line argument will be merged with vars provided via YAML config.\nPreviously, one overwrote the other completely\n\nPlease\nsee [Usage Notes for the account Parameter (for the connect Method)](https://docs.snowflake.com/en/user-guide/python-connector-api.html#label-account-format-info)\nfor more details on how to structure the account name.\n\n### connections.toml File\n\nA `[connections.toml](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-connect#connecting-using-the-connections-toml-file)\nfilepath can be supplied in the following ways (in order of priority):\n\n1. The `--connections-file-path` [command-line argument](#commands)\n2. The `connections-file-path` [YAML value](#yaml-config-file)\n\nA connection name can be supplied in the following ways (in order of priority):\n\n1. The `SNOWFLAKE_DEFAULT_CONNECTION_NAME` [environment variable](#environment-variables)\n2. The `--connection-name` [command-line argument](#commands)\n3. The `connection-name` [YAML value](#yaml-config-file)\n\n### YAML Config File\n\nBy default, Schemachange expects the YAML config file to be named `schemachange-config.yml`, located in the current\nworking directory. The YAML file name can be overridden with the\n`--config-file-name` [command-line argument](#commands). The folder can be overridden by using the\n`--config-folder` [command-line argument](#commands)\n\nHere is the list of available configurations in the `schemachange-config.yml` file:\n\n```yaml\nconfig-version: 1\n\n# The root folder for the database change scripts\nroot-folder: '/path/to/folder'\n\n# The modules folder for jinja macros and templates to be used across multiple scripts.\nmodules-folder: null\n\n# Override the default connections.toml file path at snowflake.connector.constants.CONNECTIONS_FILE (OS specific)\nconnections-file-path: null\n\n# Override the default connections.toml connection name. Other connection-related values will override these connection values.\nconnection-name: null\n\n# Used to override the default name of the change history table (the default is METADATA.SCHEMACHANGE.CHANGE_HISTORY)\nchange-history-table: null\n\n# Define values for the variables to replaced in change scripts. vars supplied via the command line will be merged into YAML-supplied vars\nvars:\n  var1: 'value1'\n  var2: 'value2'\n  secrets:\n    var3: 'value3' # This is considered a secret and will not be displayed in any output\n\n# Create the change history schema and table, if they do not exist (the default is False)\ncreate-change-history-table: false\n\n# Enable autocommit feature for DML commands (the default is False)\nautocommit: false\n\n# Display verbose debugging details during execution (the default is False)\nverbose: false\n\n# Run schemachange in dry run mode (the default is False)\ndry-run: false\n\n# A string to include in the QUERY_TAG that is attached to every SQL statement executed\nquery-tag: 'QUERY_TAG'\n```\n\n#### Yaml Jinja support\n\nThe YAML config file supports the jinja templating language and has a custom function \"env_var\" to access environmental\nvariables. Jinja variables are unavailable and not yet loaded since they are supplied by the YAML file. Customisation of\nthe YAML file can only happen through values passed via environment variables.\n\n##### env_var\n\nProvides access to environmental variables. The function can be used two different ways.\n\nReturn the value of the environmental variable if it exists, otherwise return the default value.\n\n```jinja\n{{ env_var('\u003cenvironmental_variable\u003e', 'default') }}\n```\n\nReturn the value of the environmental variable if it exists, otherwise raise an error.\n\n```jinja\n{{ env_var('\u003cenvironmental_variable\u003e') }}\n```\n\n## Commands\n\nSchemachange supports a few subcommands. If the subcommand is not provided it defaults to deploy. This behaviour keeps\ncompatibility with versions prior to 3.2.\n\n### deploy\n\nThis is the main command that runs the deployment process.\n\n```bash\nusage: schemachange deploy [-h] [--config-folder CONFIG_FOLDER] [--config-file-name CONFIG_FILE_NAME] [-f ROOT_FOLDER] [-m MODULES_FOLDER] [--connections-file-path CONNECTIONS_FILE_PATH] [--connection-name CONNECTION_NAME] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG]\n```\n\n| Parameter                                                            | Description                                                                                                                                                                                                                                                         |\n|----------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|\n| -h, --help                                                           | Show the help message and exit                                                                                                                                                                                                                                      |\n| --config-folder CONFIG_FOLDER                                        | The folder to look in for the schemachange config file (the default is the current working directory)                                                                                                                                                               |\n| --config-file-name CONFIG_FILE_NAME                                  | The file name of the schemachange config file. (the default is schemachange-config.yml)                                                                                                                                                                             |\n| -f ROOT_FOLDER, --root-folder ROOT_FOLDER                            | The root folder for the database change scripts. The default is the current directory.                                                                                                                                                                              |\n| -m MODULES_FOLDER, --modules-folder MODULES_FOLDER                   | The modules folder for jinja macros and templates to be used across mutliple scripts                                                                                                                                                                                |\n| --connections-file-path CONNECTIONS_FILE_PATH                        | Override the default [connections.toml](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-connect#connecting-using-the-connections-toml-file) file path at snowflake.connector.constants.CONNECTIONS_FILE (OS specific)               |\n| --connection-name CONNECTION_NAME                                    | Override the default [connections.toml](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-connect#connecting-using-the-connections-toml-file) connection name. Other connection-related values will override these connection values. |\n| -c CHANGE_HISTORY_TABLE, --change-history-table CHANGE_HISTORY_TABLE | Used to override the default name of the change history table (which is METADATA.SCHEMACHANGE.CHANGE_HISTORY)                                                                                                                                                       |\n| --vars VARS                                                          | Define values for the variables to replaced in change scripts, given in JSON format. Vars supplied via the command line will be merged with YAML-supplied vars (e.g. '{\"variable1\": \"value1\", \"variable2\": \"value2\"}')                                              |\n| --create-change-history-table                                        | Create the change history table if it does not exist. The default is 'False'.                                                                                                                                                                                       |\n| -ac, --autocommit                                                    | Enable autocommit feature for DML commands. The default is 'False'.                                                                                                                                                                                                 |\n| -v, --verbose                                                        | Display verbose debugging details during execution. The default is 'False'.                                                                                                                                                                                         |\n| --dry-run                                                            | Run schemachange in dry run mode. The default is 'False'.                                                                                                                                                                                                           |\n| --query-tag                                                          | A string to include in the QUERY_TAG that is attached to every SQL statement executed.                                                                                                                                                                              |\n\n### render\n\nThis subcommand is used to render a single script to the console. It is intended to support the development and\ntroubleshooting of script that use features from the jinja template engine.\n\n`usage: schemachange render [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-m MODULES_FOLDER] [--vars VARS] [-v] script`\n\n| Parameter                                          | Description                                                                                                                               |\n|----------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------|\n| --config-folder CONFIG_FOLDER                      | The folder to look in for the schemachange-config.yml file (the default is the current working directory)                                 |\n| -f ROOT_FOLDER, --root-folder ROOT_FOLDER          | The root folder for the database change scripts                                                                                           |\n| -m MODULES_FOLDER, --modules-folder MODULES_FOLDER | The modules folder for jinja macros and templates to be used across multiple scripts                                                      |\n| --vars VARS                                        | Define values for the variables to replaced in change scripts, given in JSON format (e.g. {\"variable1\": \"value1\", \"variable2\": \"value2\"}) |\n| -v, --verbose                                      | Display verbose debugging details during execution (the default is False)                                                                 |\n\n## Running schemachange\n\n### Prerequisites\n\nIn order to run schemachange you must have the following:\n\n* You will need to have a recent version of python 3 installed\n* You will need to have the\n  latest [Snowflake Python driver installed](https://docs.snowflake.com/en/user-guide/python-connector-install.html)\n* You will need to create the change history table used by schemachange in Snowflake (\n  see [Change History Table](#change-history-table) above for more details)\n    * First, you will need to create a database to store your change history table (schemachange will not help you with\n      this). For your convenience, [initialize.sql file](demo/provision/initialize.sql) has been provided to get you\n      started. Feel free to align the script to your organizations RBAC implementation.\n      The [setup_schemachange_schema.sql](demo/provision/setup_schemachange_schema.sql) file is provided to set up the\n      target schema that will host the change history table for each of the demo projects in this repo. Use it as a\n      means to test the required permissions and connectivity in your local setup.\n    * Second, you will need to create the change history schema and table. You can do this manually (\n      see [Change History Table](#change-history-table) above for the DDL) or have schemachange create them by running\n      it with the `--create-change-history-table` parameter (just make sure the Snowflake user you're running\n      schemachange with has privileges to create a schema and table in that database)\n* You will need to create (or choose) a user account that has privileges to apply the changes in your change script\n    * Don't forget that this user also needs the SELECT and INSERT privileges on the change history table\n\n### Running the Script\n\nschemachange is a single python script located at [schemachange/cli.py](schemachange/cli.py). It can be executed as\nfollows:\n\n```\npython schemachange/cli.py [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] [--connections-file-path] [--connection-name]\n```\n\nOr if installed via `pip`, it can be executed as follows:\n\n```\nschemachange [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] [--connections-file-path] [--connection-name]\n```\n\nThe [demo](demo) folder in this project repository contains three schemachange demo projects for you to try out. These\ndemos showcase the basics and a couple of advanced examples based on the standard Snowflake Citibike demo which can be\nfound in [the Snowflake Hands-on Lab](https://docs.snowflake.net/manuals/other-resources.html#hands-on-lab). Check out\neach demo listed below\n\n- [Basics Demo](demo/basics_demo): Used to test the basic schemachange functionality.\n- [Citibike Demo](demo/citibike_demo): Used to show a simple example of building a database and loading data using\n  schemachange.\n- [Citibike Jinja Demo](demo/citibike_demo_jinja): Extends the citibike demo to showcase the use of macros and jinja\n  templating.\n\nThe [Citibike data](https://www.citibikenyc.com/system-data) for this demo comes from the NYC Citi Bike bike share\nprogram.\n\nTo get started with schemachange and these demo scripts follow these steps:\n\n1. Make sure you've completed the [Prerequisites](#prerequisites) steps above\n1. Get a copy of this schemachange repository (either via a clone or download)\n1. Open a shell and change directory to your copy of the schemachange repository\n1. Run schemachange (see [Running the Script](#running-the-script) above) with your Snowflake account details and\n   respective demo project as the root folder (make sure you use the full path)\n\n## Integrating With DevOps\n\n### Sample DevOps Process Flow\n\nHere is a sample DevOps development lifecycle with schemachange:\n\n\u003cimg src=\"https://github.com/user-attachments/assets/42eae968-ae76-4fcb-a0ba-3995ec977818\" alt=\"schemachange DevOps process\" title=\"schemachange DevOps process\" /\u003e\n\n### Using in a CI/CD Pipeline\n\nIf your build agent has a recent version of python 3 installed, the script can be run like so:\n\n```bash\npip install schemachange --upgrade\nschemachange [-h] [-f ROOT_FOLDER] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] [--connections-file-path] [--connection-name]\n```\n\nOr if you prefer docker, run like so:\n\n```bash\ndocker run -it --rm \\\n  --name schemachange-script \\\n  -v \"$PWD\":/usr/src/schemachange \\\n  -w /usr/src/schemachange \\\n  -e ROOT_FOLDER \\\n  -e $CONNECTION_NAME \\\n  python:3 /bin/bash -c \"pip install schemachange --upgrade \u0026\u0026 schemachange -f $ROOT_FOLDER --connections-file-path connections.toml --connection-name $CONNECTION_NAME\"\n```\n\nEither way, don't forget to configure a [connections.toml file](#connectionstoml-file) for connection parameters\n\n## Maintainers\n\n- James Weakley (@jamesweakley)\n- Jeremiah Hansen (@jeremiahhansen)\n\nThis is a community-developed tool, not an official Snowflake offering. It comes with no support or warranty. However,\nfeel free to raise a GitHub issue if you find a bug or would like a new feature.\n\n## Third Party Packages\n\nThe current functionality in schemachange would not be possible without the following third party packages and all those\nthat maintain and have contributed.\n\n| Name                       | License                 | Author                                                                                                           | URL                                  |\n|----------------------------|-------------------------|------------------------------------------------------------------------------------------------------------------|--------------------------------------|\n| Jinja2                     | BSD License             | Armin Ronacher                                                                                                   | https://palletsprojects.com/p/jinja/ |\n| PyYAML                     | MIT License             | Kirill Simonov                                                                                                   | https://pyyaml.org/                  |\n| pandas                     | BSD License             | The Pandas Development Team                                                                                      | https://pandas.pydata.org            |\n| pytest                     | MIT License             | Holger Krekel, Bruno Oliveira, Ronny Pfannschmidt, Floris Bruynooghe, Brianna Laugher, Florian Bruhin and others | https://docs.pytest.org/en/latest/   |\n| snowflake-connector-python | Apache Software License | Snowflake, Inc                                                                                                   | https://www.snowflake.com/           |\n\n## Legal\n\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this tool except in compliance with the\nLicense. You may obtain a copy of the License\nat: [http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0)\n\nUnless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"\nAS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\nlanguage governing permissions and limitations under the License.\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fsnowflake-labs%2Fschemachange","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fsnowflake-labs%2Fschemachange","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fsnowflake-labs%2Fschemachange/lists"}