From 334a5bf354ac964a14f2e2dc3851ed4d853b28d7 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 7 Nov 2024 11:26:57 -0500 Subject: [PATCH] Update CHANGELOG.md for `0.52.0` release, add scripts/ instructions for ASF releases (#1479) --- .github/workflows/rust.yml | 35 ----- .gitignore | 1 + CHANGELOG.md | 4 +- Cargo.toml | 14 +- changelog/0.52.0.md | 104 ++++++++++++++ dev/release/README.md | 181 ++++++++++++++++++++++++ dev/release/check-rat-report.py | 59 ++++++++ dev/release/create-tarball.sh | 135 ++++++++++++++++++ dev/release/generate-changelog.py | 164 +++++++++++++++++++++ dev/release/rat_exclude_files.txt | 6 + dev/release/release-tarball.sh | 74 ++++++++++ dev/release/run-rat.sh | 43 ++++++ dev/release/verify-release-candidate.sh | 152 ++++++++++++++++++++ docs/releasing.md | 81 ----------- 14 files changed, 926 insertions(+), 127 deletions(-) create mode 100644 changelog/0.52.0.md create mode 100644 dev/release/README.md create mode 100644 dev/release/check-rat-report.py create mode 100755 dev/release/create-tarball.sh create mode 100755 dev/release/generate-changelog.py create mode 100644 dev/release/rat_exclude_files.txt create mode 100755 dev/release/release-tarball.sh create mode 100755 dev/release/run-rat.sh create mode 100755 dev/release/verify-release-candidate.sh delete mode 100644 docs/releasing.md diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 253d8ab27..2502abe9d 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -85,38 +85,3 @@ jobs: use-tool-cache: true - name: Test run: cargo test --all-features - - test-coverage: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Setup Rust Toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Install Tarpaulin - uses: actions-rs/install@v0.1 - with: - crate: cargo-tarpaulin - version: 0.14.2 - use-tool-cache: true - - name: Coverage - run: cargo tarpaulin -o Lcov --output-dir ./coverage - - name: Coveralls - uses: coverallsapp/github-action@master - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - publish-crate: - if: startsWith(github.ref, 'refs/tags/v0') - runs-on: ubuntu-latest - needs: [test] - steps: - - uses: actions/checkout@v4 - - name: Setup Rust Toolchain - uses: ./.github/actions/setup-builder - - name: Publish - shell: bash - run: | - cargo publish --token ${{ secrets.CRATES_TOKEN }} diff --git a/.gitignore b/.gitignore index 4c6821d47..f705d0b0d 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ /target/ /sqlparser_bench/target/ /derive/target/ +dev/dist # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries # More information here http://doc.crates.io/guide.html#cargotoml-vs-cargolock diff --git a/CHANGELOG.md b/CHANGELOG.md index e047515ad..ec74bf633 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,5 +24,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm Given that the parser produces a typed AST, any changes to the AST will technically be breaking and thus will result in a `0.(N+1)` version. + - Unreleased: Check https://github.com/sqlparser-rs/sqlparser-rs/commits/main for undocumented changes. -- `0.51.0` and earlier: [changelog/0.51.0-pre.md](changelog/0.51.0-pre.md) \ No newline at end of file +- `0.52.0`: [changelog/0.52.0.md](changelog/0.52.0.md) +- `0.51.0` and earlier: [changelog/0.51.0-pre.md](changelog/0.51.0-pre.md) diff --git a/Cargo.toml b/Cargo.toml index 99546465b..18b246e04 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,12 +18,12 @@ [package] name = "sqlparser" description = "Extensible SQL Lexer and Parser with support for ANSI SQL:2011" -version = "0.51.0" -authors = ["Andy Grove "] -homepage = "https://github.com/sqlparser-rs/sqlparser-rs" +version = "0.52.0" +authors = ["Apache DataFusion "] +homepage = "https://github.com/apache/datafusion-sqlparser-rs" documentation = "https://docs.rs/sqlparser/" keywords = ["ansi", "sql", "lexer", "parser"] -repository = "https://github.com/sqlparser-rs/sqlparser-rs" +repository = "https://github.com/apache/datafusion-sqlparser-rs" license = "Apache-2.0" include = [ "src/**/*.rs", @@ -58,12 +58,6 @@ simple_logger = "5.0" matches = "0.1" pretty_assertions = "1" -[package.metadata.release] -# Instruct `cargo release` to not run `cargo publish` locally: -# https://github.com/sunng87/cargo-release/blob/master/docs/reference.md#config-fields -# See docs/releasing.md for details. -publish = false - [package.metadata.docs.rs] # Document these features on docs.rs features = ["serde", "visitor"] diff --git a/changelog/0.52.0.md b/changelog/0.52.0.md new file mode 100644 index 000000000..9d5b16c7c --- /dev/null +++ b/changelog/0.52.0.md @@ -0,0 +1,104 @@ + + +# sqlparser-rs 0.52.0 Changelog + +This release consists of 45 commits from 20 contributors. See credits at the end of this changelog for more information. + +**Implemented enhancements:** + +- feat: support explain options [#1426](https://github.com/apache/datafusion-sqlparser-rs/pull/1426) (kysshsy) +- feat: adding Display implementation to DELETE and INSERT [#1427](https://github.com/apache/datafusion-sqlparser-rs/pull/1427) (seve-martinez) + +**Fixed bugs:** + +- fix: `maybe_parse` preventing parser from erroring on recursion limit [#1464](https://github.com/apache/datafusion-sqlparser-rs/pull/1464) (tomershaniii) + +**Other:** + +- Fix parsing of negative values [#1419](https://github.com/apache/datafusion-sqlparser-rs/pull/1419) (agscpp) +- Allow to use ON CLUSTER cluster_name in TRUNCATE syntax [#1428](https://github.com/apache/datafusion-sqlparser-rs/pull/1428) (git-hulk) +- chore: remove redundant punctuation [#1434](https://github.com/apache/datafusion-sqlparser-rs/pull/1434) (Fischer0522) +- MS SQL Server: add support for IDENTITY column option [#1432](https://github.com/apache/datafusion-sqlparser-rs/pull/1432) (7phs) +- Update to ASF header / add when missing [#1437](https://github.com/apache/datafusion-sqlparser-rs/pull/1437) (alamb) +- Some small optimizations [#1424](https://github.com/apache/datafusion-sqlparser-rs/pull/1424) (exrok) +- Fix `codestyle` CI check [#1438](https://github.com/apache/datafusion-sqlparser-rs/pull/1438) (alamb) +- Implements CREATE POLICY syntax for PostgreSQL [#1440](https://github.com/apache/datafusion-sqlparser-rs/pull/1440) (git-hulk) +- make `parse_expr_with_alias` public [#1444](https://github.com/apache/datafusion-sqlparser-rs/pull/1444) (Eason0729) +- Implements DROP POLICY syntax for PostgreSQL [#1445](https://github.com/apache/datafusion-sqlparser-rs/pull/1445) (git-hulk) +- Support `DROP DATABASE` [#1443](https://github.com/apache/datafusion-sqlparser-rs/pull/1443) (linhr) +- Implements ALTER POLICY syntax for PostgreSQL [#1446](https://github.com/apache/datafusion-sqlparser-rs/pull/1446) (git-hulk) +- Add a note discouraging new use of `dialect_of` macro [#1448](https://github.com/apache/datafusion-sqlparser-rs/pull/1448) (alamb) +- Expand handling of `LIMIT 1, 2` handling to include sqlite [#1447](https://github.com/apache/datafusion-sqlparser-rs/pull/1447) (joshuawarner32) +- Fix always uses CommentDef::WithoutEq while parsing the inline comment [#1453](https://github.com/apache/datafusion-sqlparser-rs/pull/1453) (git-hulk) +- Add support for the LIKE ANY and ILIKE ANY pattern-matching condition [#1456](https://github.com/apache/datafusion-sqlparser-rs/pull/1456) (yoavcloud) +- added ability to parse extension to parse_comment inside postgres dialect [#1451](https://github.com/apache/datafusion-sqlparser-rs/pull/1451) (MaxwellKnight) +- Snowflake: support of views column comment [#1441](https://github.com/apache/datafusion-sqlparser-rs/pull/1441) (7phs) +- Add SQLite "ON CONFLICT" column option in CREATE TABLE statements [#1442](https://github.com/apache/datafusion-sqlparser-rs/pull/1442) (nucccc) +- Add support for ASC and DESC in CREATE TABLE column constraints for SQLite. [#1462](https://github.com/apache/datafusion-sqlparser-rs/pull/1462) (caldwell) +- Add support of `EXPLAIN QUERY PLAN` syntax for SQLite dialect [#1458](https://github.com/apache/datafusion-sqlparser-rs/pull/1458) (git-hulk) +- Add "DROP TYPE" support. [#1461](https://github.com/apache/datafusion-sqlparser-rs/pull/1461) (caldwell) +- chore: Add asf.yaml [#1463](https://github.com/apache/datafusion-sqlparser-rs/pull/1463) (Xuanwo) +- Add support for quantified comparison predicates (ALL/ANY/SOME) [#1459](https://github.com/apache/datafusion-sqlparser-rs/pull/1459) (yoavcloud) +- MySQL dialect: Add support for hash comments [#1466](https://github.com/apache/datafusion-sqlparser-rs/pull/1466) (hansott) +- Fix #1469 (SET ROLE regression) [#1474](https://github.com/apache/datafusion-sqlparser-rs/pull/1474) (lovasoa) +- Add support for parsing MsSql alias with equals [#1467](https://github.com/apache/datafusion-sqlparser-rs/pull/1467) (yoavcloud) +- Snowflake: support for extended column options in `CREATE TABLE` [#1454](https://github.com/apache/datafusion-sqlparser-rs/pull/1454) (7phs) +- MsSQL TRY_CONVERT [#1477](https://github.com/apache/datafusion-sqlparser-rs/pull/1477) (yoavcloud) +- Add PostgreSQL specfic "CREATE TYPE t AS ENUM (...)" support. [#1460](https://github.com/apache/datafusion-sqlparser-rs/pull/1460) (caldwell) +- Fix build [#1483](https://github.com/apache/datafusion-sqlparser-rs/pull/1483) (yoavcloud) +- Fix complex blocks warning when running clippy [#1488](https://github.com/apache/datafusion-sqlparser-rs/pull/1488) (git-hulk) +- Add support for SHOW DATABASES/SCHEMAS/TABLES/VIEWS in Hive [#1487](https://github.com/apache/datafusion-sqlparser-rs/pull/1487) (yoavcloud) +- Fix typo in `Dialect::supports_eq_alias_assigment` [#1478](https://github.com/apache/datafusion-sqlparser-rs/pull/1478) (alamb) +- Add support for PostgreSQL `LISTEN/NOTIFY` syntax [#1485](https://github.com/apache/datafusion-sqlparser-rs/pull/1485) (wugeer) +- Add support for TOP before ALL/DISTINCT [#1495](https://github.com/apache/datafusion-sqlparser-rs/pull/1495) (yoavcloud) +- add support for `FOR ORDINALITY` and `NESTED` in JSON_TABLE [#1493](https://github.com/apache/datafusion-sqlparser-rs/pull/1493) (lovasoa) +- Add Apache License to additional files [#1502](https://github.com/apache/datafusion-sqlparser-rs/pull/1502) (alamb) +- Move CHANGELOG content [#1503](https://github.com/apache/datafusion-sqlparser-rs/pull/1503) (alamb) +- improve support for T-SQL EXECUTE statements [#1490](https://github.com/apache/datafusion-sqlparser-rs/pull/1490) (lovasoa) + +## Credits + +Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. + +``` + 8 Andrew Lamb + 7 Yoav Cohen + 7 hulk + 3 Aleksei Piianin + 3 David Caldwell + 3 Ophir LOJKINE + 1 Agaev Guseyn + 1 Eason + 1 Fischer + 1 Hans Ott + 1 Heran Lin + 1 Joshua Warner + 1 Maxwell Knight + 1 Seve Martinez + 1 Siyuan Huang + 1 Thomas Dagenais + 1 Xuanwo + 1 nucccc + 1 tomershaniii + 1 wugeer +``` + +Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. + diff --git a/dev/release/README.md b/dev/release/README.md new file mode 100644 index 000000000..c440f7387 --- /dev/null +++ b/dev/release/README.md @@ -0,0 +1,181 @@ + + + +## Process Overview + +As part of the Apache governance model, official releases consist of signed +source tarballs approved by the DataFusion PMC. + +We then use the code in the approved artifacts to release to crates.io. + +### Change Log + +We maintain a `CHANGELOG.md` so our users know what has been changed between releases. + +You will need a GitHub Personal Access Token for the following steps. Follow +[these instructions](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) +to generate one if you do not already have one. + +The changelog is generated using a Python script which needs `PyGitHub`, installed using pip: + +```shell +pip3 install PyGitHub +``` + +To generate the changelog, set the `GITHUB_TOKEN` environment variable to a valid token and then run the script +providing two commit ids or tags followed by the version number of the release being created. The following +example generates a change log of all changes between the first commit and the current HEAD revision. + +```shell +export GITHUB_TOKEN= +python ./dev/release/generate-changelog.py v0.51.0 HEAD 0.52.0 > changelog/0.52.0.md +``` + +This script creates a changelog from GitHub PRs based on the labels associated with them as well as looking for +titles starting with `feat:`, `fix:`, or `docs:`. + +Add an entry to CHANGELOG.md for the new version + +## Prepare release commits and PR + +### Update Version + +Checkout the main commit to be released + +```shell +git fetch apache +git checkout apache/main +``` + +Manually update the version in the root `Cargo.toml` to the release version (e.g. `0.52.0`). + +Lastly commit the version change: + +```shell +git commit -a -m 'Update version' +``` + +## Prepare release candidate artifacts + +After the PR gets merged, you are ready to create release artifacts from the +merged commit. + +(Note you need to be a committer to run these scripts as they upload to the apache svn distribution servers) + +### Pick a Release Candidate (RC) number + +Pick numbers in sequential order, with `0` for `rc0`, `1` for `rc1`, etc. + +### Create git tag for the release: + +While the official release artifacts are signed tarballs and zip files, we also +tag the commit it was created for convenience and code archaeology. + +Using a string such as `v0.52.0` as the ``, create and push the tag by running these commands: + +For example to tag version `0.52.0` + +```shell +git fetch apache +git tag v0.52.0-rc1 apache/main +# push tag to Github remote +git push apache v0.52.0-rc1 +``` + +### Create, sign, and upload artifacts + +Run `create-tarball.sh` with the `` tag and `` and you found in previous steps: + +```shell +GITHUB_TOKEN= ./dev/release/create-tarball.sh 0.52.0 1 +``` + +The `create-tarball.sh` script + +1. creates and uploads all release candidate artifacts to the [datafusion + dev](https://dist.apache.org/repos/dist/dev/datafusion) location on the + apache distribution svn server + +2. provide you an email template to + send to dev@datafusion.apache.org for release voting. + +### Vote on Release Candidate artifacts + +Send the email output from the script to dev@datafusion.apache.org. + +For the release to become "official" it needs at least three PMC members to vote +1 on it. + +### Verifying Release Candidates + +The `dev/release/verify-release-candidate.sh` is a script in this repository that can assist in the verification process. Run it like: + +```shell +./dev/release/verify-release-candidate.sh 0.52.0 1 +``` + +#### If the release is not approved + +If the release is not approved, fix whatever the problem is, merge changelog +changes into main if there is any and try again with the next RC number. + +## Finalize the release + +NOTE: steps in this section can only be done by PMC members. + +### After the release is approved + +Move artifacts to the release location in SVN, using the `release-tarball.sh` script: + +```shell +./dev/release/release-tarball.sh 0.52.0 1 +``` + +Congratulations! The release is now official! + +### Publish on Crates.io + +Only approved releases of the tarball should be published to +crates.io, in order to conform to Apache Software Foundation +governance standards. + +A DataFusion committer can publish this crate after an official project release has +been made to crates.io using the following instructions. + +Follow [these +instructions](https://doc.rust-lang.org/cargo/reference/publishing.html) to +create an account and login to crates.io before asking to be added as an owner +to the sqlparser DataFusion crates. + +Download and unpack the official release tarball + +Verify that the Cargo.toml in the tarball contains the correct version +(e.g. `version = "0.52.0"`) and then publish the crates by running the following commands + +```shell +cargo publish +``` + +If necessary, also publish the `sqlparser_derive` crate: + +crates.io homepage: https://crates.io/crates/sqlparser_derive + +```shell +(cd derive && cargo publish +``` diff --git a/dev/release/check-rat-report.py b/dev/release/check-rat-report.py new file mode 100644 index 000000000..e30d72bdd --- /dev/null +++ b/dev/release/check-rat-report.py @@ -0,0 +1,59 @@ +#!/usr/bin/python +############################################################################## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +############################################################################## +import fnmatch +import re +import sys +import xml.etree.ElementTree as ET + +if len(sys.argv) != 3: + sys.stderr.write("Usage: %s exclude_globs.lst rat_report.xml\n" % + sys.argv[0]) + sys.exit(1) + +exclude_globs_filename = sys.argv[1] +xml_filename = sys.argv[2] + +globs = [line.strip() for line in open(exclude_globs_filename, "r")] + +tree = ET.parse(xml_filename) +root = tree.getroot() +resources = root.findall('resource') + +all_ok = True +for r in resources: + approvals = r.findall('license-approval') + if not approvals or approvals[0].attrib['name'] == 'true': + continue + clean_name = re.sub('^[^/]+/', '', r.attrib['name']) + excluded = False + for g in globs: + if fnmatch.fnmatch(clean_name, g): + excluded = True + break + if not excluded: + sys.stdout.write("NOT APPROVED: %s (%s): %s\n" % ( + clean_name, r.attrib['name'], approvals[0].attrib['name'])) + all_ok = False + +if not all_ok: + sys.exit(1) + +print('OK') +sys.exit(0) diff --git a/dev/release/create-tarball.sh b/dev/release/create-tarball.sh new file mode 100755 index 000000000..4cb17cd36 --- /dev/null +++ b/dev/release/create-tarball.sh @@ -0,0 +1,135 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Adapted from https://github.com/apache/datafusion/tree/master/dev/release/create-tarball.sh + +# This script creates a signed tarball in +# dev/dist/apache-datafusion-sqlparser-rs--rc.tar.gz and uploads it to +# the "dev" area of the dist.apache.datafusion repository and prepares an +# email for sending to the dev@datafusion.apache.org list for a formal +# vote. +# +# See release/README.md for full release instructions +# +# Requirements: +# +# 1. gpg setup for signing and have uploaded your public +# signature to https://pgp.mit.edu/ +# +# 2. Logged into the apache svn server with the appropriate +# credentials +# +# 3. Install the requests python package +# +# +# Based in part on 02-source.sh from apache/arrow +# + +set -e + +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)" + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + echo "ex. $0 0.52.0 2" + exit +fi + +if [[ -z "${GITHUB_TOKEN}" ]]; then + echo "Please set personal github token through GITHUB_TOKEN environment variable" + exit +fi + +version=$1 +rc=$2 +tag="v${version}-rc${rc}" + +echo "Attempting to create ${tarball} from tag ${tag}" +release_hash=$(cd "${SOURCE_TOP_DIR}" && git rev-list --max-count=1 ${tag}) + +release=apache-datafusion-sqlparser-rs-${version} +distdir=${SOURCE_TOP_DIR}/dev/dist/${release}-rc${rc} +tarname=${release}.tar.gz +tarball=${distdir}/${tarname} +url="https://dist.apache.org/repos/dist/dev/datafusion/${release}-rc${rc}" + +if [ -z "$release_hash" ]; then + echo "Cannot continue: unknown git tag: ${tag}" +fi + +echo "Draft email for dev@datafusion.apache.org mailing list" +echo "" +echo "---------------------------------------------------------" +cat < containing the files in git at $release_hash +# the files in the tarball are prefixed with {version} (e.g. 4.0.1) +mkdir -p ${distdir} +(cd "${SOURCE_TOP_DIR}" && git archive ${release_hash} --prefix ${release}/ | gzip > ${tarball}) + +echo "Running rat license checker on ${tarball}" +${SOURCE_DIR}/run-rat.sh ${tarball} + +echo "Signing tarball and creating checksums" +gpg --armor --output ${tarball}.asc --detach-sig ${tarball} +# create signing with relative path of tarball +# so that they can be verified with a command such as +# shasum --check apache-datafusion-sqlparser-rs-0.52.0-rc1.tar.gz.sha512 +(cd ${distdir} && shasum -a 256 ${tarname}) > ${tarball}.sha256 +(cd ${distdir} && shasum -a 512 ${tarname}) > ${tarball}.sha512 + + +echo "Uploading to sqlparser-rs dist/dev to ${url}" +svn co --depth=empty https://dist.apache.org/repos/dist/dev/datafusion ${SOURCE_TOP_DIR}/dev/dist +svn add ${distdir} +svn ci -m "Apache DataFusion ${version} ${rc}" ${distdir} diff --git a/dev/release/generate-changelog.py b/dev/release/generate-changelog.py new file mode 100755 index 000000000..52fd2e548 --- /dev/null +++ b/dev/release/generate-changelog.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import sys +from github import Github +import os +import re +import subprocess + +def print_pulls(repo_name, title, pulls): + if len(pulls) > 0: + print("**{}:**".format(title)) + print() + for (pull, commit) in pulls: + url = "https://github.com/{}/pull/{}".format(repo_name, pull.number) + print("- {} [#{}]({}) ({})".format(pull.title, pull.number, url, commit.author.login)) + print() + + +def generate_changelog(repo, repo_name, tag1, tag2, version): + + # get a list of commits between two tags + print(f"Fetching list of commits between {tag1} and {tag2}", file=sys.stderr) + comparison = repo.compare(tag1, tag2) + + # get the pull requests for these commits + print("Fetching pull requests", file=sys.stderr) + unique_pulls = [] + all_pulls = [] + for commit in comparison.commits: + pulls = commit.get_pulls() + for pull in pulls: + # there can be multiple commits per PR if squash merge is not being used and + # in this case we should get all the author names, but for now just pick one + if pull.number not in unique_pulls: + unique_pulls.append(pull.number) + all_pulls.append((pull, commit)) + + # we split the pulls into categories + breaking = [] + bugs = [] + docs = [] + enhancements = [] + performance = [] + other = [] + + # categorize the pull requests based on GitHub labels + print("Categorizing pull requests", file=sys.stderr) + for (pull, commit) in all_pulls: + + # see if PR title uses Conventional Commits + cc_type = '' + cc_scope = '' + cc_breaking = '' + parts = re.findall(r'^([a-z]+)(\([a-z]+\))?(!)?:', pull.title) + if len(parts) == 1: + parts_tuple = parts[0] + cc_type = parts_tuple[0] # fix, feat, docs, chore + cc_scope = parts_tuple[1] # component within project + cc_breaking = parts_tuple[2] == '!' + + labels = [label.name for label in pull.labels] + if 'api change' in labels or cc_breaking: + breaking.append((pull, commit)) + elif 'bug' in labels or cc_type == 'fix': + bugs.append((pull, commit)) + elif 'performance' in labels or cc_type == 'perf': + performance.append((pull, commit)) + elif 'enhancement' in labels or cc_type == 'feat': + enhancements.append((pull, commit)) + elif 'documentation' in labels or cc_type == 'docs' or cc_type == 'doc': + docs.append((pull, commit)) + else: + other.append((pull, commit)) + + # produce the changelog content + print("Generating changelog content", file=sys.stderr) + + # ASF header + print("""\n""") + + print(f"# sqlparser-rs {version} Changelog\n") + + # get the number of commits + commit_count = subprocess.check_output(f"git log --pretty=oneline {tag1}..{tag2} | wc -l", shell=True, text=True).strip() + + # get number of contributors + contributor_count = subprocess.check_output(f"git shortlog -sn {tag1}..{tag2} | wc -l", shell=True, text=True).strip() + + print(f"This release consists of {commit_count} commits from {contributor_count} contributors. " + f"See credits at the end of this changelog for more information.\n") + + print_pulls(repo_name, "Breaking changes", breaking) + print_pulls(repo_name, "Performance related", performance) + print_pulls(repo_name, "Implemented enhancements", enhancements) + print_pulls(repo_name, "Fixed bugs", bugs) + print_pulls(repo_name, "Documentation updates", docs) + print_pulls(repo_name, "Other", other) + + # show code contributions + credits = subprocess.check_output(f"git shortlog -sn {tag1}..{tag2}", shell=True, text=True).rstrip() + + print("## Credits\n") + print("Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) " + "per contributor.\n") + print("```") + print(credits) + print("```\n") + + print("Thank you also to everyone who contributed in other ways such as filing issues, reviewing " + "PRs, and providing feedback on this release.\n") + +def cli(args=None): + """Process command line arguments.""" + if not args: + args = sys.argv[1:] + + parser = argparse.ArgumentParser() + parser.add_argument("tag1", help="The previous commit or tag (e.g. 0.1.0)") + parser.add_argument("tag2", help="The current commit or tag (e.g. HEAD)") + parser.add_argument("version", help="The version number to include in the changelog") + args = parser.parse_args() + + token = os.getenv("GITHUB_TOKEN") + project = "apache/datafusion-sqlparser-rs" + + g = Github(token) + repo = g.get_repo(project) + generate_changelog(repo, project, args.tag1, args.tag2, args.version) + +if __name__ == "__main__": + cli() \ No newline at end of file diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt new file mode 100644 index 000000000..a567eda9c --- /dev/null +++ b/dev/release/rat_exclude_files.txt @@ -0,0 +1,6 @@ +# Files to exclude from the Apache Rat (license) check +.gitignore +.tool-versions +dev/release/rat_exclude_files.txt +fuzz/.gitignore + diff --git a/dev/release/release-tarball.sh b/dev/release/release-tarball.sh new file mode 100755 index 000000000..e59b2776c --- /dev/null +++ b/dev/release/release-tarball.sh @@ -0,0 +1,74 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Adapted from https://github.com/apache/arrow-rs/tree/master/dev/release/release-tarball.sh + +# This script copies a tarball from the "dev" area of the +# dist.apache.datafusion repository to the "release" area +# +# This script should only be run after the release has been approved +# by the Apache DataFusion PMC committee. +# +# See release/README.md for full release instructions +# +# Based in part on post-01-upload.sh from apache/arrow + + +set -e +set -u + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + echo "ex. $0 0.52.0 2" + exit +fi + +version=$1 +rc=$2 + +tmp_dir=tmp-apache-datafusion-dist + +echo "Recreate temporary directory: ${tmp_dir}" +rm -rf ${tmp_dir} +mkdir -p ${tmp_dir} + +echo "Clone dev dist repository" +svn \ + co \ + https://dist.apache.org/repos/dist/dev/datafusion/apache-datafusion-sqlparser-rs-${version}-rc${rc} \ + ${tmp_dir}/dev + +echo "Clone release dist repository" +svn co https://dist.apache.org/repos/dist/release/datafusion ${tmp_dir}/release + +echo "Copy ${version}-rc${rc} to release working copy" +release_version=datafusion-sqlparser-rs-${version} +mkdir -p ${tmp_dir}/release/${release_version} +cp -r ${tmp_dir}/dev/* ${tmp_dir}/release/${release_version}/ +svn add ${tmp_dir}/release/${release_version} + +echo "Commit release" +svn ci -m "Apache DataFusion sqlparser-rs ${version}" ${tmp_dir}/release + +echo "Clean up" +rm -rf ${tmp_dir} + +echo "Success! The release is available here:" +echo " https://dist.apache.org/repos/dist/release/datafusion/${release_version}" diff --git a/dev/release/run-rat.sh b/dev/release/run-rat.sh new file mode 100755 index 000000000..94fa55fbe --- /dev/null +++ b/dev/release/run-rat.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +RAT_VERSION=0.13 + +# download apache rat +if [ ! -f apache-rat-${RAT_VERSION}.jar ]; then + curl -s https://repo1.maven.org/maven2/org/apache/rat/apache-rat/${RAT_VERSION}/apache-rat-${RAT_VERSION}.jar > apache-rat-${RAT_VERSION}.jar +fi + +RAT="java -jar apache-rat-${RAT_VERSION}.jar -x " + +RELEASE_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd) + +# generate the rat report +$RAT $1 > rat.txt +python $RELEASE_DIR/check-rat-report.py $RELEASE_DIR/rat_exclude_files.txt rat.txt > filtered_rat.txt +cat filtered_rat.txt +UNAPPROVED=`cat filtered_rat.txt | grep "NOT APPROVED" | wc -l` + +if [ "0" -eq "${UNAPPROVED}" ]; then + echo "No unapproved licenses" +else + echo "${UNAPPROVED} unapproved licences. Check rat report: rat.txt" + exit 1 +fi diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh new file mode 100755 index 000000000..9ff7e17b5 --- /dev/null +++ b/dev/release/verify-release-candidate.sh @@ -0,0 +1,152 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +case $# in + 2) VERSION="$1" + RC_NUMBER="$2" + ;; + *) echo "Usage: $0 X.Y.Z RC_NUMBER" + exit 1 + ;; +esac + +set -e +set -x +set -o pipefail + +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +ARROW_DIR="$(dirname $(dirname ${SOURCE_DIR}))" +ARROW_DIST_URL='https://dist.apache.org/repos/dist/dev/datafusion' + +download_dist_file() { + curl \ + --silent \ + --show-error \ + --fail \ + --location \ + --remote-name $ARROW_DIST_URL/$1 +} + +download_rc_file() { + download_dist_file apache-datafusion-sqlparser-rs-${VERSION}-rc${RC_NUMBER}/$1 +} + +import_gpg_keys() { + download_dist_file KEYS + gpg --import KEYS +} + +if type shasum >/dev/null 2>&1; then + sha256_verify="shasum -a 256 -c" + sha512_verify="shasum -a 512 -c" +else + sha256_verify="sha256sum -c" + sha512_verify="sha512sum -c" +fi + +fetch_archive() { + local dist_name=$1 + download_rc_file ${dist_name}.tar.gz + download_rc_file ${dist_name}.tar.gz.asc + download_rc_file ${dist_name}.tar.gz.sha256 + download_rc_file ${dist_name}.tar.gz.sha512 + verify_dir_artifact_signatures +} + +verify_dir_artifact_signatures() { + # verify the signature and the checksums of each artifact + find . -name '*.asc' | while read sigfile; do + artifact=${sigfile/.asc/} + gpg --verify $sigfile $artifact || exit 1 + + # go into the directory because the checksum files contain only the + # basename of the artifact + pushd $(dirname $artifact) + base_artifact=$(basename $artifact) + ${sha256_verify} $base_artifact.sha256 || exit 1 + ${sha512_verify} $base_artifact.sha512 || exit 1 + popd + done +} + +setup_tempdir() { + cleanup() { + if [ "${TEST_SUCCESS}" = "yes" ]; then + rm -fr "${ARROW_TMPDIR}" + else + echo "Failed to verify release candidate. See ${ARROW_TMPDIR} for details." + fi + } + + if [ -z "${ARROW_TMPDIR}" ]; then + # clean up automatically if ARROW_TMPDIR is not defined + ARROW_TMPDIR=$(mktemp -d -t "$1.XXXXX") + trap cleanup EXIT + else + # don't clean up automatically + mkdir -p "${ARROW_TMPDIR}" + fi +} + +test_source_distribution() { + # install rust toolchain in a similar fashion like test-miniconda + export RUSTUP_HOME=$PWD/test-rustup + export CARGO_HOME=$PWD/test-rustup + + curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path + + export PATH=$RUSTUP_HOME/bin:$PATH + source $RUSTUP_HOME/env + + # build and test rust + + # raises on any formatting errors + rustup component add rustfmt --toolchain stable + cargo fmt --all -- --check + + cargo build + cargo test --all-features + + if ( find -iname 'Cargo.toml' | xargs grep SNAPSHOT ); then + echo "Cargo.toml version should not contain SNAPSHOT for releases" + exit 1 + fi + + # Check that publish works + cargo publish --dry-run +} + +TEST_SUCCESS=no + +setup_tempdir "datafusion-sqlparser-rs-${VERSION}" +echo "Working in sandbox ${ARROW_TMPDIR}" +cd ${ARROW_TMPDIR} + +dist_name="apache-datafusion-sqlparser-rs-${VERSION}" +import_gpg_keys +fetch_archive ${dist_name} +tar xf ${dist_name}.tar.gz +pushd ${dist_name} + test_source_distribution +popd + +TEST_SUCCESS=yes +echo 'Release candidate looks good!' +exit 0 diff --git a/docs/releasing.md b/docs/releasing.md deleted file mode 100644 index c1b85a20c..000000000 --- a/docs/releasing.md +++ /dev/null @@ -1,81 +0,0 @@ - - -# Releasing - -## Prerequisites -Publishing to crates.io has been automated via GitHub Actions, so you will only -need push access to the [sqlparser-rs GitHub repository](https://github.com/sqlparser-rs/sqlparser-rs) -in order to publish a release. - -We use the [`cargo release`](https://github.com/sunng87/cargo-release) -subcommand to ensure correct versioning. Install via: - -``` -$ cargo install cargo-release -``` - -## Process - -1. **Before releasing** ensure `CHANGELOG.md` is updated appropriately and that - you have a clean checkout of the `main` branch of the sqlparser repository: - ``` - $ git fetch && git status - On branch main - Your branch is up to date with 'origin/main'. - - nothing to commit, working tree clean - ``` - * If you have the time, check that the examples in the README are up to date. - -2. Using `cargo-release` we can publish a new release like so: - - ``` - $ cargo release minor --push-remote origin - ``` - - After verifying, you can rerun with `--execute` if all looks good. - You can add `--no-push` to stop before actually publishing the release. - - `cargo release` will then: - - * Bump the minor part of the version in `Cargo.toml` (e.g. `0.7.1-alpha.0` - -> `0.8.0`. You can use `patch` instead of `minor`, as appropriate). - * Create a new tag (e.g. `v0.8.0`) locally - * Push the new tag to the specified remote (`origin` in the above - example), which will trigger a publishing process to crates.io as part of - the [corresponding GitHub Action](https://github.com/sqlparser-rs/sqlparser-rs/blob/main/.github/workflows/rust.yml). - - Note that credentials for authoring in this way are securely stored in - the (GitHub) repo secrets as `CRATE_TOKEN`. - -4. Check that the new version of the crate is available on crates.io: - https://crates.io/crates/sqlparser - - -## `sqlparser_derive` crate - -Currently this crate is manually published via `cargo publish`. - -crates.io homepage: https://crates.io/crates/sqlparser_derive - -```shell -cd derive -cargo publish -```