GH-3708: Inline parquet.thrift

dev/check-parquet-thrift-release.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Verifies that the parquet-format.version sidecar file is tracking a released
+# parquet-format version and that the inlined parquet.thrift byte-matches the
+# upstream file at that version.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+THRIFT_FILE="${REPO_ROOT}/parquet-format-structures/src/main/thrift/parquet.thrift"
+SIDECAR_FILE="${REPO_ROOT}/parquet-format-structures/src/main/thrift/parquet-format.version"
+
+# shellcheck source=parquet-thrift-lib.sh
+source "${SCRIPT_DIR}/parquet-thrift-lib.sh"
+
+if [[ ! -f "$SIDECAR_FILE" ]]; then
+  echo "ERROR: sidecar not found: ${SIDECAR_FILE}" >&2
+  exit 1
+fi
+
+if [[ ! -f "$THRIFT_FILE" ]]; then
+  echo "ERROR: inlined thrift not found: ${THRIFT_FILE}" >&2
+  exit 1
+fi
+
+version="$(grep '^parquet-format.version=' "$SIDECAR_FILE" | cut -d= -f2 || true)"
+commit="$(grep '^parquet-format.commit=' "$SIDECAR_FILE" | cut -d= -f2 || true)"
+
+# Require a valid X.Y.Z release version
+if [[ -z "$version" ]] || ! [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+  echo "ERROR: inlined parquet.thrift is not from a released parquet-format version." >&2
+  echo "  parquet-format.version = '${version}'" >&2
+  echo "" >&2
+  echo "A parquet-java release must use a released parquet-format IDL." >&2
+  echo "Run: dev/update-parquet-thrift.sh --version <X.Y.Z>" >&2
+  exit 1
+fi
+
+echo "Checking inlined parquet.thrift against parquet-format ${version} (commit ${commit}) ..."
+
+# Fetch the canonical IDL from the release tag. Never writes to tracked files.
+# Fails closed: a network/git failure or an empty file all exit non-zero.
+tmp="$(mktemp)"
+trap 'rm -f "$tmp"' EXIT
+
+if ! fetch_thrift "apache-parquet-format-${version}" "$tmp"; then
+  echo "ERROR: failed to fetch parquet.thrift for tag apache-parquet-format-${version}" >&2
+  echo "Cannot verify inlined IDL — failing closed." >&2
+  exit 1
+fi
+
+if [[ ! -s "$tmp" ]]; then
+  echo "ERROR: fetched parquet.thrift is empty for tag apache-parquet-format-${version}" >&2
+  echo "Cannot verify inlined IDL — failing closed." >&2
+  exit 1
+fi
+
+if diff -u "$tmp" "$THRIFT_FILE"; then
+  echo "OK: inlined parquet.thrift matches parquet-format ${version}."
+  exit 0
+else
+  echo "" >&2
+  echo "ERROR: inlined parquet.thrift differs from parquet-format ${version}." >&2
+  echo "Run: dev/update-parquet-thrift.sh --version ${version}" >&2
+  exit 1
+fi

dev/parquet-thrift-lib.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Shared helpers for update-parquet-thrift.sh and check-parquet-thrift-release.sh.
+# Source this file; do not execute it. No top-level side effects, no `exit` calls.
+
+PARQUET_FORMAT_REPO="https://github.com/apache/parquet-format"
+PARQUET_FORMAT_RAW="https://raw.githubusercontent.com/apache/parquet-format"
+THRIFT_PATH_IN_FORMAT="src/main/thrift/parquet.thrift"
+
+# Resolve a parquet-format release version to its full 40-char commit sha using
+# git ls-remote. Prefers the dereferenced ref (^{}) so it works for annotated
+# and lightweight tags. Echoes the sha and returns 0 on success; returns 1 if
+# the tag does not exist; returns git's exit code (and lets git's stderr through)
+# on git/network failure.
+resolve_version_to_commit() {
+  local ver="$1" sha ls_out ls_rc
+  ls_out="$(git ls-remote --tags "$PARQUET_FORMAT_REPO" \
+              "refs/tags/apache-parquet-format-${ver}" \
+              "refs/tags/apache-parquet-format-${ver}^{}")" || { ls_rc=$?; return $ls_rc; }
+  sha="$(printf '%s\n' "$ls_out" \
+         | awk '{ if ($2 ~ /\^\{\}$/) deref=$1; else plain=$1 }
+                END { print (deref != "" ? deref : plain) }')"
+  [[ -n "$sha" ]] || return 1
+  printf '%s\n' "$sha"
+}
+
+# Fetch the parquet.thrift IDL from the parquet-format repo at the given commit
+# sha or tag, writing output to $2. Reads from raw.githubusercontent.com, which
+# serves any commit sha or tag and is not subject to the GitHub REST API rate
+# limit. Returns 0 on success, 1 on any failure.
+# The caller is responsible for providing a temp path for $2 so tracked source
+# files are never overwritten on failure.
+fetch_thrift() {
+  local ref="$1" dest="$2" url attempt
+  url="${PARQUET_FORMAT_RAW}/${ref}/${THRIFT_PATH_IN_FORMAT}"
+  for attempt in 1 2 3; do
+    if curl -fsSL -o "$dest" "$url" && [[ -s "$dest" ]]; then
+      return 0
+    fi
+    if [[ "$attempt" -eq 3 ]]; then
+      echo "ERROR: failed to fetch ${url} after ${attempt} attempts" >&2
+      return 1
+    fi
+    sleep $((attempt * 2))
+  done
+}

dev/prepare-release.sh

@@ -38,6 +38,9 @@ new_development_version="$release_version-SNAPSHOT"
 
 tag="apache-parquet-$release_version-rc$2"
 
+# Ensure the inlined parquet.thrift matches an official parquet-format release.
+"$(dirname "$0")/check-parquet-thrift-release.sh"
+
 ./mvnw release:clean
 ./mvnw release:prepare -DskipTests -Darguments=-DskipTests -Dtag="$tag" "-DreleaseVersion=$release_version" -DdevelopmentVersion="$new_development_version"
 

dev/update-parquet-thrift.sh

@@ -0,0 +1,165 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Updates the inlined parquet.thrift and sidecar to a specific parquet-format
+# commit or release.
+#
+# Usage:
+#   update-parquet-thrift.sh <full-40-char-commit-hash>  -- POC: sets version=UNRELEASED
+#   update-parquet-thrift.sh --version <X.Y.Z>           -- release sync: derives commit from tag
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+THRIFT_FILE="${REPO_ROOT}/parquet-format-structures/src/main/thrift/parquet.thrift"
+SIDECAR_FILE="${REPO_ROOT}/parquet-format-structures/src/main/thrift/parquet-format.version"
+
+# shellcheck source=parquet-thrift-lib.sh
+source "${SCRIPT_DIR}/parquet-thrift-lib.sh"
+
+usage() {
+  cat <<EOF
+Usage:
+  $0 <full-40-char-commit-hash>  POC sync — fetch parquet.thrift at that commit; version=UNRELEASED
+  $0 --version <X.Y.Z>           Release sync — resolve tag apache-parquet-format-<X.Y.Z>
+
+Only one of a commit hash or --version may be given, not both.
+Commit mode requires a full 40-character sha (short shas cannot be fetched from a remote).
+EOF
+  exit 1
+}
+
+if [[ $# -eq 0 ]]; then
+  usage
+fi
+
+mode=""
+commit_arg=""
+version_arg=""
+
+if [[ "$1" == "--version" ]]; then
+  [[ $# -ne 2 ]] && usage
+  mode="version"
+  version_arg="$2"
+elif [[ "$1" == --* ]]; then
+  usage
+else
+  [[ $# -ne 1 ]] && usage
+  mode="commit"
+  commit_arg="$1"
+fi
+
+# Validate arguments
+if [[ "$mode" == "version" ]]; then
+  if ! [[ "$version_arg" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+    echo "ERROR: version must be X.Y.Z (e.g. 2.13.0), got: $version_arg" >&2
+    exit 1
+  fi
+else
+  # Full 40-char sha required — short shas cannot be fetched from a remote (git exits 128).
+  if ! [[ "$commit_arg" =~ ^[0-9a-f]{40}$ ]]; then
+    echo "ERROR: commit hash must be exactly 40 hex chars, got: $commit_arg" >&2
+    echo "       Obtain the full sha with: git ls-remote ${PARQUET_FORMAT_REPO} HEAD" >&2
+    exit 1
+  fi
+fi
+
+# Resolve the commit sha to use
+resolved_sha=""
+resolved_version=""
+
+if [[ "$mode" == "version" ]]; then
+  echo "Resolving tag apache-parquet-format-${version_arg} ..."
+  resolved_sha="$(resolve_version_to_commit "$version_arg")" || {
+    rc=$?
+    if [[ $rc -eq 1 ]]; then
+      echo "ERROR: tag apache-parquet-format-${version_arg} not found in ${PARQUET_FORMAT_REPO}" >&2
+      echo "       Check that the version exists: git ls-remote --tags ${PARQUET_FORMAT_REPO}" >&2
+    else
+      echo "ERROR: git/network failure resolving tag apache-parquet-format-${version_arg}" >&2
+    fi
+    exit 1
+  }
+  resolved_version="$version_arg"
+  echo "  -> commit: $resolved_sha"
+else
+  resolved_sha="$commit_arg"
+  resolved_version="UNRELEASED"
+fi
+
+# Read the old commit from the sidecar (for the summary)
+old_commit="(none)"
+if [[ -f "$SIDECAR_FILE" ]]; then
+  old_commit="$(grep '^parquet-format.commit=' "$SIDECAR_FILE" | cut -d= -f2 || true)"
+fi
+
+# Fetch parquet.thrift into a temp file. Only overwrite the tracked file after
+# the fetch and validation succeed, so a failure never deletes tracked source.
+tmp_thrift="$(mktemp)"
+trap 'rm -f "$tmp_thrift"' EXIT
+
+echo "Fetching parquet.thrift at ${resolved_sha} ..."
+if ! fetch_thrift "$resolved_sha" "$tmp_thrift"; then
+  echo "ERROR: could not fetch parquet.thrift at ${resolved_sha}" >&2
+  echo "       The tracked files were not modified." >&2
+  exit 1
+fi
+
+# Validate: non-empty and contains the expected namespace declaration.
+if [[ ! -s "$tmp_thrift" ]]; then
+  echo "ERROR: fetched parquet.thrift is empty" >&2
+  echo "       The tracked files were not modified." >&2
+  exit 1
+fi
+if ! grep -q 'namespace java org.apache.parquet.format' "$tmp_thrift"; then
+  echo "ERROR: fetched file does not look like parquet.thrift (missing namespace declaration)" >&2
+  echo "       The tracked files were not modified." >&2
+  exit 1
+fi
+
+# Commit the validated thrift file, then write the sidecar. Order matters:
+# if the mv fails, the sidecar is not yet updated, keeping the two in sync.
+mv "$tmp_thrift" "$THRIFT_FILE"
+chmod 644 "$THRIFT_FILE"
+echo "  -> written to ${THRIFT_FILE}"
+
+source_url="${PARQUET_FORMAT_REPO}/blob/${resolved_sha}/src/main/thrift/parquet.thrift"
+cat > "$SIDECAR_FILE" <<SIDECAR
+# Provenance of the inlined parquet.thrift. Maintained by dev/update-parquet-thrift.sh.
+parquet-format.commit=${resolved_sha}
+parquet-format.version=${resolved_version}
+parquet-format.source=${source_url}
+SIDECAR
+echo "  -> sidecar updated: ${SIDECAR_FILE}"
+
+# Summary
+echo ""
+echo "Update complete:"
+echo "  old commit: ${old_commit}"
+echo "  new commit: ${resolved_sha}"
+if [[ "$resolved_version" == "UNRELEASED" ]]; then
+  echo "  version:    UNRELEASED (POC sync — run --version <ver> for a release sync)"
+else
+  echo "  version:    ${resolved_version} (release sync)"
+fi
+echo ""
+echo "Next steps: rebuild parquet-format-structures and review the diff:"
+echo "  ./mvnw -pl parquet-format-structures -am install -DskipTests"
+echo "  git diff parquet-format-structures/src/main/thrift/parquet.thrift"