Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 98 additions & 11 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
Expand Down Expand Up @@ -57,12 +57,15 @@
cpp: ${{ steps.filter.outputs.cpp }}
cpp_code: ${{ steps.filter.outputs.cpp_code }}
java_code: ${{ steps.filter.outputs.java_code }}
android: ${{ steps.filter.outputs.android }}
python: ${{ steps.filter.outputs.python }}
graalvm: ${{ steps.filter.outputs.graalvm }}
graalvm_kotlin: ${{ steps.filter.outputs.graalvm_kotlin }}
rust: ${{ steps.filter.outputs.rust }}
swift: ${{ steps.filter.outputs.swift }}
javascript: ${{ steps.filter.outputs.javascript }}
kotlin: ${{ steps.filter.outputs.kotlin }}
kotlin_benchmark: ${{ steps.filter.outputs.kotlin_benchmark }}
scala: ${{ steps.filter.outputs.scala }}
steps:
- uses: actions/checkout@v5
Expand All @@ -78,12 +81,15 @@
echo "cpp=true" >> "$GITHUB_OUTPUT"
echo "cpp_code=true" >> "$GITHUB_OUTPUT"
echo "java_code=true" >> "$GITHUB_OUTPUT"
echo "android=true" >> "$GITHUB_OUTPUT"
echo "python=true" >> "$GITHUB_OUTPUT"
echo "graalvm=true" >> "$GITHUB_OUTPUT"
echo "graalvm_kotlin=true" >> "$GITHUB_OUTPUT"
echo "rust=true" >> "$GITHUB_OUTPUT"
echo "swift=true" >> "$GITHUB_OUTPUT"
echo "javascript=true" >> "$GITHUB_OUTPUT"
echo "kotlin=true" >> "$GITHUB_OUTPUT"
echo "kotlin_benchmark=true" >> "$GITHUB_OUTPUT"
echo "scala=true" >> "$GITHUB_OUTPUT"
exit 0
fi
Expand Down Expand Up @@ -121,6 +127,12 @@
echo "java_code=false" >> "$GITHUB_OUTPUT"
fi

if grep -Eq '^(\.github/workflows/ci\.yml$|ci/run_ci\.(py|sh)$|ci/tasks/(common|kotlin)\.py$|java/(pom\.xml$|fory-(core|json|annotation-processor)/)|kotlin/|integration_tests/(kotlin_json_corpus|android_tests)/)' <<< "$changed_files"; then
echo "android=true" >> "$GITHUB_OUTPUT"
else
echo "android=false" >> "$GITHUB_OUTPUT"
fi

if grep -Eq '^(python/)' <<< "$changed_files"; then
echo "python=true" >> "$GITHUB_OUTPUT"
else
Expand All @@ -133,6 +145,12 @@
echo "graalvm=false" >> "$GITHUB_OUTPUT"
fi

if grep -Eq '^(\.github/workflows/ci\.yml$|ci/run_ci\.(py|sh)$|ci/tasks/(common|kotlin)\.py$|java/(pom\.xml$|fory-(core|json|annotation-processor)/)|kotlin/|integration_tests/(kotlin_json_corpus|graalvm_kotlin_tests)/)' <<< "$changed_files"; then
echo "graalvm_kotlin=true" >> "$GITHUB_OUTPUT"
else
echo "graalvm_kotlin=false" >> "$GITHUB_OUTPUT"
fi

if grep -Eq '^(rust/)' <<< "$changed_files"; then
echo "rust=true" >> "$GITHUB_OUTPUT"
else
Expand All @@ -151,12 +169,18 @@
echo "javascript=false" >> "$GITHUB_OUTPUT"
fi

if grep -Eq '^(java/|kotlin/)' <<< "$changed_files"; then
if grep -Eq '^(\.github/workflows/(ci\.yml|release-jvm-snapshot\.yaml)$|ci/run_ci\.(py|sh)$|ci/tasks/(common|java|kotlin)\.py$|ci/(release|test_release)\.py$|java/(pom\.xml$|fory-(core|json|annotation-processor)/)|kotlin/|integration_tests/(kotlin_json_corpus|graalvm_kotlin_tests|jpms_tests|grpc_tests/kotlin|idl_tests/kotlin)/)' <<< "$changed_files"; then
echo "kotlin=true" >> "$GITHUB_OUTPUT"
else
echo "kotlin=false" >> "$GITHUB_OUTPUT"
fi

if grep -Eq '^(\.github/workflows/ci\.yml$|ci/run_ci\.(py|sh)$|ci/tasks/(common|kotlin)\.py$|java/(pom\.xml$|fory-(core|json|annotation-processor)/)|kotlin/|benchmarks/kotlin/)' <<< "$changed_files"; then
echo "kotlin_benchmark=true" >> "$GITHUB_OUTPUT"
else
echo "kotlin_benchmark=false" >> "$GITHUB_OUTPUT"
fi

if grep -Eq '^(java/|scala/)' <<< "$changed_files"; then
echo "scala=true" >> "$GITHUB_OUTPUT"
else
Expand Down Expand Up @@ -287,7 +311,7 @@
android:
name: Android Instrumented Tests API ${{ matrix.api-level }}
needs: changes
if: needs.changes.outputs.java_code == 'true'
if: needs.changes.outputs.android == 'true'
runs-on: ubuntu-latest
strategy:
fail-fast: false
Expand All @@ -300,6 +324,10 @@
with:
java-version: 17
distribution: "temurin"
- name: Set up Python 3.11
uses: actions/setup-python@v5
with:
python-version: 3.11
- name: Cache Maven local repository
uses: actions/cache@v4
with:
Expand All @@ -311,10 +339,8 @@
uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e
with:
gradle-version: "8.13"
- name: Install Fory Java artifacts
run: |
cd java
mvn -T16 --no-transfer-progress -pl fory-json,fory-annotation-processor -am install -DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip=true
- name: Install Fory Java and Kotlin JSON artifacts
run: python ./ci/run_ci.py kotlin --task install
- name: Enable KVM
run: |
echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
Expand Down Expand Up @@ -623,6 +649,32 @@
shell: bash
run: ./ci/run_ci.sh graalvm_json_tests

graalvm_kotlin_json:
name: GraalVM Kotlin JSON CI
needs: changes
if: needs.changes.outputs.graalvm_kotlin == 'true'
runs-on: ubuntu-latest
strategy:
matrix:
java-version: ["17", "25"]
steps:
- uses: actions/checkout@v5
- uses: graalvm/setup-graalvm@6f3fa030c4b8f77c1f554a860f593a654538fa38 # 1.5.6
with:
java-version: ${{ matrix.java-version }}
distribution: "graalvm"
github-token: ${{ secrets.GITHUB_TOKEN }}
native-image-job-reports: "true"
- name: Cache Maven local repository
uses: actions/cache@v4
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Build and run Kotlin JSON native image
run: python ./ci/run_ci.py kotlin --task native-json

kotlin:
name: Kotlin CI
needs: changes
Expand All @@ -632,7 +684,7 @@
MY_VAR: "PATH"
strategy:
matrix:
java-version: ["8", "11", "17", "21"]
java-version: ["8", "11", "17", "21", "25", "26"]
steps:
- uses: actions/checkout@v5
- name: Set up JDK ${{ matrix.java-version }}
Expand All @@ -647,15 +699,50 @@
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Set up Python 3.8
- name: Set up Python 3.11
uses: actions/setup-python@v5
with:
python-version: 3.8
- name: Install fory java
run: python ./ci/run_ci.py java --install-jdks --install-fory
python-version: 3.11
- name: Run Kotlin CI
run: python ./ci/run_ci.py kotlin

kotlin_json_benchmark:
name: Kotlin JSON Benchmark Correctness
needs: changes
if: needs.changes.outputs.kotlin_benchmark == 'true'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v5
- name: Set up JDK 17
uses: actions/setup-java@v4
with:
java-version: 17
distribution: "temurin"
- name: Set up Python 3.11
uses: actions/setup-python@v5
with:
python-version: 3.11
- name: Cache Maven local repository
uses: actions/cache@v4
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Set up Gradle 9.3.0
uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e
with:
gradle-version: "9.3.0"
- name: Install Fory Java and Kotlin JSON artifacts
run: python ./ci/run_ci.py kotlin --task install
- name: Verify benchmark correctness and generated sources
run: gradle --no-daemon -p benchmarks/kotlin test verifyGeneratedJsonArtifacts jmhClasses
- name: Install benchmark report dependencies
run: python -m pip install -r benchmarks/kotlin/requirements.txt
- name: Verify benchmark reports
working-directory: benchmarks/kotlin
run: python -m unittest test_benchmark_report.py

scala:
name: Scala CI
needs: changes
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,21 +18,24 @@
# All `uses:` action pins in this workflow must come from the Apache action allowlist:
# https://github.com/apache/infrastructure-actions/blob/main/actions.yml

name: Publish Fory Java Snapshot
name: Publish Fory JVM Snapshot

on:
push:
branches:
- main
- release-java-snapshot
- release-jvm-snapshot

permissions:
contents: read

jobs:
publish-java:
publish-jvm:
runs-on: ubuntu-latest
if: github.repository == 'apache/fory'
steps:
- uses: actions/checkout@v5
- name: Set up Maven Central Repository
- name: Set up Apache snapshot repository
uses: actions/setup-java@v4
with:
java-version: "25"
Expand All @@ -42,8 +45,11 @@ jobs:
server-id: apache.snapshots.https
server-username: NEXUS_USERNAME
server-password: NEXUS_PASSWORD
- name: Publish Fory Java Snapshot
run: python ./ci/run_ci.py java --version 25 --release
- uses: sbt/setup-sbt@9d56cf12e9b58d219605e1d8bfe69a8395fedde0 # v1.5.1
with:
disk-cache: false
- name: Publish Java, Kotlin, and Scala snapshots
run: python ./ci/release.py publish_jvm --mode snapshot
env:
NEXUS_USERNAME: ${{ secrets.NEXUS_USER }}
NEXUS_PASSWORD: ${{ secrets.NEXUS_PW }}
4 changes: 4 additions & 0 deletions benchmarks/kotlin/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
.gradle/
build/
reports/
__pycache__/
90 changes: 90 additions & 0 deletions benchmarks/kotlin/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
# Kotlin JSON Benchmarks

This standalone Gradle/JMH project compares Fory JSON Kotlin, kotlinx.serialization, Moshi, and
Jackson Kotlin on one immutable Eishay `MediaContent` model. It is a repository benchmark project,
not a published Fory artifact.

The model uses only `val` properties and has no public zero-argument constructor. All four
libraries consume the same model and fixture; no library-specific data transfer object or measured
model conversion is used. The fixture SHA-256 is
`8faba2f57ab397f319aced5cf1e8411a76785557d4c7d1703ec9d540354310a1`.

## Compared operations

The suite contains exactly 16 methods: four libraries multiplied by String serialization, UTF-8
byte serialization, String deserialization, and UTF-8 byte deserialization.

Each library uses a retained declared-type or generated serializer API:

- Fory retains `jsonTypeRef<MediaContent>()`, disables asynchronous compilation for deterministic
setup, enables null emission, and warms all four generated paths before measurement.
- kotlinx.serialization retains `MediaContent.serializer()` and uses its String and stream APIs.
- Moshi retains its KSP-generated adapter and uses its String and Okio buffer APIs.
- Jackson Kotlin retains one `ObjectReader` and one `ObjectWriter` and uses its direct String and
byte APIs.

The final byte materialization required by a library remains inside its measured byte-serialization
method. Deserialization likewise includes any fresh in-memory stream or buffer required by that
library. No byte method routes through a prebuilt String.

## Correctness gates

Before timing, setup verifies that every library:

- decodes the exact fixture from String and UTF-8 bytes to the independent expected object;
- emits structurally equivalent JSON from String and byte APIs; and
- round-trips its own String and byte output.

The Gradle build also fails unless the generated Fory companion, operation class, per-model R8
resource, and Moshi adapter are present for every object model. This prevents missing processor
output from silently changing the measured path.

## Build and run

Install the current `fory-json-kotlin` and `fory-json-kotlin-ksp` artifacts in Maven local first.
Use Gradle 9.3.0 and a JDK 17 or later toolchain. Install the pinned Python report dependencies
before running the report tests or producing charts:

```bash
python -m pip install -r requirements.txt
```

Build the correctness and JMH artifacts:

```bash
gradle --no-daemon test verifyGeneratedJsonArtifacts jmhJar writeBenchmarkClasspath
```

Run the paired process-isolated scheduler:

```bash
python run_json_benchmark.py --rounds 6 --output-dir reports/json
```

Each round launches 16 separate JVM processes and runs one exact method per process. For every
operation, Fory and one comparator are adjacent; the selected comparator and AB/BA direction rotate
across rounds. Only those adjacent AB/BA launches contribute to Fory/comparator ratios. The report
computes each ratio inside its round before calculating the median and median absolute deviation.
The round count must be a multiple of three; the six-round default gives every comparator one AB
and one BA adjacency for each operation.

Use `--prepare-only` for CI correctness and source-generation checks without performance timing.
When comparing two Fory revisions at the same Maven coordinate, resolve each revision from a
separate Maven repository and build one JMH JAR from each isolated classpath. Supply the second JMH
JAR and generated classpath manifest with `--comparison-jmh-jar` and
`--comparison-classpath-file`, plus its commit with `--comparison-commit`. The runner rejects a
shared Fory artifact path, verifies that the immutable model, fixture, benchmark methods, and JMH
case list are identical, records both artifact and dependency-set hashes plus the executed JMH JAR
hash for every launch, and alternates current/comparison Fory launches in adjacent AB/BA pairs. If
the comparison revision lacks this exact module, API, or benchmark surface, do not report a
revision ratio.

Excluded runs are retained. Use `--session-id` when predeclaring deterministic run IDs, then supply
a CSV with `run_id,reason` columns through `--exclusions`; the raw sample remains present with
`included=false` and the reason. A failed process is also retained and fails the overall run. A
completed raw CSV can instead be reviewed, marked with exclusions, and passed directly to
`benchmark_report.py` without deleting any launch.

See the [published Kotlin JSON benchmark report](../../docs/benchmarks/json/kotlin/README.md). The
published page explicitly remains pending until a complete measured run is available; the tooling
does not synthesize results.
Loading
Loading