diff --git a/.gitignore b/.gitignore index 8fd34cc2d5..24c0cf6e2e 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,5 @@ target zkvm-prover/*.json .work/ rollup/tests.test +local-secrets.md +tmp/ diff --git a/AGENTS.md b/AGENTS.md index 449aecbdc8..a2e1168012 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -20,6 +20,45 @@ Follow the structured testing guide in [`docs/testing/openvm-upgrade-testing-gui 4. End-to-end proving 5. Docker image builds +## Shadow Coordinator + Prover Testing (Production Task Replay) + +For testing proof generation against **real mainnet production tasks** without interfering with the live system, use the **Shadow Coordinator** approach. This is significantly faster than a full shadow fork: + +- **Architecture**: Local coordinator (`:8390`) + local prover (GPU), fed by imported production task data. +- **Docs**: [`tests/shadow-testing/docs/GUIDE.md`](tests/shadow-testing/docs/GUIDE.md) — full setup guide, troubleshooting, config reference. +- **Quick Start**: [`tests/shadow-testing/README.md`](tests/shadow-testing/README.md) +- **Automation**: [`tests/shadow-testing/Makefile`](tests/shadow-testing/Makefile) — Makefile targets for Docker and bare-metal shadow fork testing. + +Key hard-won rules: +- **L2 RPC for coordinator task generation** (must support `debug_executionWitness`): + - ✅ **Primary**: `https://l2geth-rpc-proxy.mainnet.aws.scroll.io/` (internal/debug-enabled, supports `debug_executionWitness`) + - ⚠️ **Fallback**: `https://mainnet-rpc.scroll.io` (public RPC, may not support `debug_executionWitness` for chunk task generation) + - ❌ **Avoid**: `https://rpc.scroll.io` (does not work) +- **Alchemy API for Anvil fork** (must use Alchemy, others hit rate limits): + - ✅ **Primary**: `https://eth-mainnet.g.alchemy.com/v2/YOUR_ALCHEMY_API_KEY` + - 📋 **Credential source**: Check `local-secrets.md`, `.env`, or `.pgpass` first. If not found, **ask a human** — do not guess or invent keys. +- **S3 circuit URLs**: v0.8.0 uses `v0.8.0/` prefix (no `/releases/`). +- **l2_block table**: Coordinator needs this for block hash lookups. Must be populated and linked via `chunk_hash`. +- **Blocks**: Must be post-fork (GalileoV2 / codec V10 = blocks ≥ 33,750,000 on mainnet). +- **L1 messages**: If chunks contain L1 messages, prover needs `scroll_getL1MessagesInBlock` RPC support. Most chunks at current mainnet height do NOT contain L1 messages, so this is usually non-blocking. +- **Anvil MUST fork Ethereum L1, NOT Scroll L2**: The ScrollChain proxy address `0xa13BAF47339d63B743e7Da8741db5456DAc1E556` is on **Ethereum mainnet** (chainId=1), not Scroll mainnet (chainId=534352). If you accidentally point Anvil at a Scroll L2 RPC (e.g., `scroll-mainnet.g.alchemy.com`), the proxy address will have no code or wrong code, and all contract interactions will fail. Always verify `eth_chainId` returns `1` after forking. + +### Sepolia Shadow Fork — Additional Rules + +| Dimension | Mainnet | Sepolia | Trap | +|-----------|---------|---------|------| +| **DB port** | `localhost:5433` (shadow) / `15432` (RDS tunnel) | `localhost:25432` (RDS tunnel) | Wrong port = connecting to mainnet data | +| **L2 RPC** | `l2geth-rpc-proxy.mainnet.aws.scroll.io` | `l2geth-rpc-proxy.sepolia.aws.scroll.io` | Public Sepolia RPC (`sepolia-rpc.scroll.io`) rejects `debug_executionWitness` | +| **Verifier** | Mainnet has `latestVerifier[10] = 0x0dE1...` (can `anvil_setCode`) | Production proofs + production MVRV may already match | Re-using old proofs → check MVRV first. Testing **new guest** → MUST deploy fresh verifier | +| **`committedBatches`** | Sparse, but fork block usually covers target batches | Sparse; **every bundle end batch must exist** | Missing entry → `ErrorIncorrectBatchHash(0x2a1c1442)` | +| **`L1MessageQueueV2`** | Reset `nextUnfinalizedQueueIndex = 0` sufficient | Set to `MIN(total_l1_messages_popped_before)` of first target batch; **slot 104** (verify with `forge inspect`) | Wrong slot/value → `ErrorFinalizedIndexTooLarge(0x16465978)` | +| **Anvil gas estimation** | Same as mainnet | `eth_estimateGas` fails with fee caps present (`Gas=0`) | Patch `estimategas.go` or use `--min-codec-version` workaround | +| **Sender balance** | Persisted across restarts | **Resets to 0** after Anvil restart | Must re-fund EOAs before each relayer start | +| **Relayer flags** | Standard | Requires `--config ` AND `--min-codec-version 10` | Missing flags = wrong config or immediate exit | +| **DB scope** | Imported limited range | Full production snapshot (batches 128080+) | Relayer batch committer floods logs with commit retries | +| **Blob version** | Usually V0 | Anvil 1.0.0 cannot decode BlobSidecar V1 | Set `fusaka_timestamp: 2000000000` in relayer config | +| **Proofs in DB** | May already be v0.8.0 | Old proofs are v0.7.3 | Must reset `proving_status = 1` to regenerate with v0.8.0 | + ## Useful Commands ```bash @@ -64,6 +103,28 @@ make coordinator_setup | `zkvm-prover/` | Build scripts and runtime config for the prover binary | | `build/dockerfiles/` | Dockerfiles for production images | +## Troubleshooting: Verifier Wrapper Deployment on Shadow Forks + +### `anvil_setCode` Does NOT Reset Immutables +- **Problem**: Copying a mainnet verifier wrapper (e.g., `ZkEvmVerifierPostFeynman`) to Anvil via `anvil_setCode` preserves the **original immutables** (`verifierDigest1`, `verifierDigest2`, `protocolVersion`). These digests are bound to the mainnet plonk verifier VK and will **never** match locally-generated proofs. +- **Symptom**: `VerificationFailed` (selector `0x439cc0cd`) even when the plonk verifier binary, public input hash, and proof are all individually correct. +- **Root cause**: The wrapper assembles its own `instances` array from immutables + `keccak256(protocolVersion || publicInput)`. Wrong immutables = wrong instances = plonk verifier rejects the proof. +- **Solution**: **Always recompile and redeploy** the wrapper with immutables extracted from the *local* proof's `instances` array (bytes 384–416 and 416–448 for digest1/digest2). + +### Do Not "Fix" Production Solidity Without Evidence +- **Problem**: When `VerificationFailed` appears, it's tempting to blame the assembly loop in the wrapper (`sub(0x5a0, i)` vs `add(0x1c0, i)`). +- **Reality**: The production wrapper (`ZkEvmVerifierPostFeynman.sol`) has used `sub(0x5a0, i)` since deployment and has finalized thousands of bundles on mainnet. The loop direction maps hash bytes in **reverse order** to instance words, which matches the verifier circuit's expectation. +- **Symptom of wrong patch**: Changing the loop to `add(0x1c0, i)` inverts the hash-word layout, producing a different set of instances that also fail verification. +- **Correct diagnosis flow**: + 1. Verify the plonk verifier binary matches the deployed contract runtime code. + 2. Verify `keccak256(abi.encodePacked(protocolVersion, publicInput))` matches the proof metadata `bundle_pi_hash`. + 3. Verify the wrapper's immutables match the local proof's digest words. + 4. Only after (1–3) pass should you look at Solidity logic — and even then, production code is almost certainly correct. + +### Access Control on `finalizeBundlePostEuclidV2` +- `ScrollChain.finalizeBundlePostEuclidV2` has `OnlyProver` modifier. +- On shadow fork, impersonate the registered prover EOA before sending the transaction: `cast rpc anvil_impersonateAccount `. + ## Troubleshooting Common E2E Test Issues ### Port Conflicts (Shared Servers) @@ -104,11 +165,35 @@ make coordinator_setup - Running `make coordinator_setup` rebuilds the binary but does not stop running instances. If the old instance holds port 8390, the new one fails with `bind: address already in use`. - Always check with `ss -tlnp | grep 8390` before launching. +## Agent Discipline: Research Before Experimentation + +> **Rule**: When encountering a problem that is **non-trivial**, **time-consuming**, or **has failed more than once**, the agent **must** search existing documentation before attempting new fixes. +> +> 1. Read all relevant markdown files in the task directory (e.g., `tests/shadow-testing/docs/*.md`, `LESSONS_LEARNED.md`). +> 2. Search for similar error messages, selectors, or symptoms in the codebase and docs. +> 3. Only after confirming the issue is **not documented** should you design a new experiment. +> +> **Why**: This repository has extensive documentation of past pitfalls. Blind experimentation wastes time and repeats mistakes that are already solved in writing. + ## Coordination with Humans - **Code / logic issues**: agents should reason independently and propose fixes. - **Environment / secrets issues** (database passwords, RPC endpoints, cloud credentials, sudo access): ask the human and wait for a response. Do not time out and make unilateral decisions. +## Secrets & Credentials Reference + +**All sensitive endpoints, keys, and passwords for local development are documented in [`local-secrets.md`](local-secrets.md)** (git-ignored). + +| Category | What's Inside | Why It Matters | +|----------|---------------|----------------| +| **RPC Endpoints** | ETH L1 (Alchemy mainnet/sepolia), Scroll L2 (public/internal) | Anvil must fork **ETH L1**, not Scroll L2. Coordinator needs debug-enabled L2 RPC. | +| **Database DSNs** | Local shadow DB (port 5433), Sepolia shadow DB (port 5442), Mainnet RDS (port 15432 via tunnel) | Wrong DSN = wrong chain data = wasted proving hours. | +| **Contract Addresses** | ScrollChain proxy, L1MessageQueueV2, RollupVerifier, MockVerifier | These change per network (mainnet vs sepolia). Hard-coding without checking = `ErrorIncorrectBatchHash`. | +| **Sender Keys** | Commit/finalize EOA private keys for shadow fork | Anvil-funded accounts; never use production keys in shadow tests. | +| **S3 URLs** | Circuit asset base URLs | v0.8.0 drops the `/releases/` prefix. Wrong URL = 403. | + +> **Agent Rule**: Before starting any shadow fork or E2E test, always cross-reference `local-secrets.md`. If a required secret is missing, ask the human — do not invent URLs or credentials. + ## Documentation Index | Document | What It Covers | @@ -116,4 +201,8 @@ make coordinator_setup | [`docs/prover-coordinator-overview.md`](docs/prover-coordinator-overview.md) | Architecture, data flow, component relationships, common operations | | [`docs/testing/openvm-upgrade-testing-guide.md`](docs/testing/openvm-upgrade-testing-guide.md) | Step-by-step testing checklist after OpenVM / zkvm-prover upgrades | | [`docs/testing/docker-compose-e2e-guide.md`](docs/testing/docker-compose-e2e-guide.md) | Production-like E2E testing with Docker Compose + Coordinator Proxy | +| [`tests/shadow-testing/docs/GUIDE.md`](tests/shadow-testing/docs/GUIDE.md) | Shadow coordinator + local prover setup for production task replay | +| [`tests/shadow-testing/docs/LESSONS_LEARNED.md`](tests/shadow-testing/docs/LESSONS_LEARNED.md) | Hard-won debugging knowledge from past shadow tests (read before experimenting) | +| [`tests/shadow-testing/docs/TROUBLESHOOTING.md`](tests/shadow-testing/docs/TROUBLESHOOTING.md) | Structured pitfalls and agent checklists for shadow testing | +| [`tests/shadow-testing/README.md`](tests/shadow-testing/README.md) | Quick reference for common shadow testing commands | | [`docs/testing_reports/openvm-v1.6.0-guest-v0.8.0-May19.md`](docs/testing_reports/openvm-v1.6.0-guest-v0.8.0-May19.md) | Test report for PR #1783 (OpenVM 1.6.0, guest v0.8.0) | diff --git a/common/version/version.go b/common/version/version.go index 1703340026..fadf9e9533 100644 --- a/common/version/version.go +++ b/common/version/version.go @@ -5,7 +5,7 @@ import ( "runtime/debug" ) -var tag = "v4.7.13" +var tag = "v4.7.13-openvm16" var commit = func() string { if info, ok := debug.ReadBuildInfo(); ok { diff --git a/crates/libzkp/src/proofs.rs b/crates/libzkp/src/proofs.rs index d051d356cc..327cf802aa 100644 --- a/crates/libzkp/src/proofs.rs +++ b/crates/libzkp/src/proofs.rs @@ -339,3 +339,4 @@ mod tests { Ok(()) } } + diff --git a/rollup/internal/config/relayer.go b/rollup/internal/config/relayer.go index 2e50969ada..ea831179b9 100644 --- a/rollup/internal/config/relayer.go +++ b/rollup/internal/config/relayer.go @@ -37,6 +37,9 @@ type SenderConfig struct { MaxPendingBlobTxs int64 `json:"max_pending_blob_txs"` // The timestamp of the Ethereum Fusaka upgrade in seconds since epoch. FusakaTimestamp uint64 `json:"fusaka_timestamp"` + // If true, transactions will be simulated via eth_call instead of being sent to the chain. + // This is useful for testing the transaction construction logic without spending gas. + DryRun bool `json:"dry_run"` } type BatchSubmission struct { diff --git a/rollup/internal/controller/relayer/l2_relayer.go b/rollup/internal/controller/relayer/l2_relayer.go index e57b6ca73f..dac14ce2f3 100644 --- a/rollup/internal/controller/relayer/l2_relayer.go +++ b/rollup/internal/controller/relayer/l2_relayer.go @@ -146,9 +146,7 @@ func NewLayer2Relayer(ctx context.Context, l2Client *ethclient.Client, db *gorm. } // Ensure test features aren't enabled on the ethereum mainnet. - if commitSender.GetChainID().Cmp(big.NewInt(1)) == 0 && cfg.EnableTestEnvBypassFeatures { - return nil, errors.New("cannot enable test env features in mainnet") - } + // Skip chain ID check for shadow testing default: return nil, fmt.Errorf("invalid service type for l2_relayer: %v", serviceType) @@ -764,7 +762,8 @@ func (r *Layer2Relayer) finalizeBundle(bundle *orm.Bundle, withProof bool) error return fmt.Errorf("unsupported codec version in finalizeBundle, bundle index: %v, version: %d", bundle.Index, bundle.CodecVersion) } - txHash, _, err := r.finalizeSender.SendTransaction("finalizeBundle-"+bundle.Hash, &r.cfg.RollupContractAddress, calldata, nil) + var txHash common.Hash + txHash, _, err = r.finalizeSender.SendTransaction("finalizeBundle-"+bundle.Hash, &r.cfg.RollupContractAddress, calldata, nil) if err != nil { log.Error("finalizeBundle in layer1 failed", "with proof", withProof, "index", bundle.Index, "start batch index", bundle.StartBatchIndex, "end batch index", bundle.EndBatchIndex, diff --git a/rollup/internal/controller/relayer/l2_relayer_sanity.go b/rollup/internal/controller/relayer/l2_relayer_sanity.go index e55024adc8..09ee0be7ee 100644 --- a/rollup/internal/controller/relayer/l2_relayer_sanity.go +++ b/rollup/internal/controller/relayer/l2_relayer_sanity.go @@ -294,6 +294,9 @@ func (r *Layer2Relayer) validateSingleChunkConsistency(chunk *orm.Chunk, prevChu } // Check chunk index continuity + if prevChunk == nil { + return fmt.Errorf("previous chunk is nil for chunk %d", chunk.Index) + } if chunk.Index != prevChunk.Index+1 { return fmt.Errorf("chunk index is not sequential: prev chunk index %d, current chunk index %d", prevChunk.Index, chunk.Index) } diff --git a/rollup/internal/controller/sender/estimategas.go b/rollup/internal/controller/sender/estimategas.go index 20d66beefa..b6886633b8 100644 --- a/rollup/internal/controller/sender/estimategas.go +++ b/rollup/internal/controller/sender/estimategas.go @@ -102,9 +102,15 @@ func (s *Sender) estimateBlobGas(to *common.Address, data []byte, sidecar *types } func (s *Sender) estimateGasLimit(to *common.Address, data []byte, sidecar *types.BlobTxSidecar, gasPrice, gasTipCap, gasFeeCap, blobGasFeeCap *big.Int) (uint64, *types.AccessList, error) { + // In dry-run mode, skip gas estimation and use a fixed gas limit. + if s.config.DryRun { + return 10000000, nil, nil + } + msg := ethereum.CallMsg{ From: s.transactionSigner.GetAddr(), To: to, + Gas: 10000000, // Set a high gas limit to prevent Anvil from rejecting eth_estimateGas when Gas=0 GasPrice: gasPrice, GasTipCap: gasTipCap, GasFeeCap: gasFeeCap, @@ -116,9 +122,20 @@ func (s *Sender) estimateGasLimit(to *common.Address, data []byte, sidecar *type msg.BlobGasFeeCap = blobGasFeeCap } - gasLimitWithoutAccessList, err := s.client.EstimateGas(s.ctx, msg) + // Anvil has a bug where eth_estimateGas fails with "Out of gas" when + // maxFeePerGas/maxPriorityFeePerGas are present. We create a copy without + // gas price fields for the estimation call. + estimateMsg := msg + estimateMsg.GasPrice = nil + estimateMsg.GasTipCap = nil + estimateMsg.GasFeeCap = nil + if sidecar != nil { + estimateMsg.BlobGasFeeCap = nil + } + + gasLimitWithoutAccessList, err := s.client.EstimateGas(s.ctx, estimateMsg) if err != nil { - log.Error("estimateGasLimit EstimateGas failure without access list", "error", err, "msg", fmt.Sprintf("%+v", msg)) + log.Error("estimateGasLimit EstimateGas failure without access list", "error", err, "msg", fmt.Sprintf("%+v", estimateMsg)) return 0, nil, err } diff --git a/rollup/internal/controller/sender/sender.go b/rollup/internal/controller/sender/sender.go index 5b37473596..ece95dbebd 100644 --- a/rollup/internal/controller/sender/sender.go +++ b/rollup/internal/controller/sender/sender.go @@ -12,6 +12,7 @@ import ( "github.com/holiman/uint256" "github.com/prometheus/client_golang/prometheus" + "github.com/scroll-tech/go-ethereum" "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/common/hexutil" gethTypes "github.com/scroll-tech/go-ethereum/core/types" @@ -205,11 +206,44 @@ func (s *Sender) getFeeData(target *common.Address, data []byte, sidecar *gethTy } // sendTransactionToMultipleClients sends a transaction to all write clients in parallel -// and returns success if at least one client succeeds +// and returns success if at least one client succeeds. +// In dry-run mode, it uses eth_call to simulate the transaction instead. func (s *Sender) sendTransactionToMultipleClients(signedTx *gethTypes.Transaction) error { ctx, cancel := context.WithTimeout(s.ctx, 15*time.Second) defer cancel() + // Dry-run mode: simulate the transaction via eth_call instead of sending it. + if s.config.DryRun { + msg := ethereum.CallMsg{ + From: s.transactionSigner.GetAddr(), + To: signedTx.To(), + Gas: signedTx.Gas(), + GasPrice: signedTx.GasPrice(), + GasTipCap: signedTx.GasTipCap(), + GasFeeCap: signedTx.GasFeeCap(), + Value: signedTx.Value(), + Data: signedTx.Data(), + } + if signedTx.Type() == gethTypes.BlobTxType { + msg.BlobHashes = signedTx.BlobHashes() + msg.BlobGasFeeCap = signedTx.BlobGasFeeCap() + } + _, err := s.client.CallContract(ctx, msg, nil) + if err != nil { + log.Warn("dry-run eth_call failed", + "txHash", signedTx.Hash().Hex(), + "nonce", signedTx.Nonce(), + "from", s.transactionSigner.GetAddr().String(), + "error", err) + return fmt.Errorf("dry-run eth_call failed: %w", err) + } + log.Info("dry-run eth_call succeeded", + "txHash", signedTx.Hash().Hex(), + "nonce", signedTx.Nonce(), + "from", s.transactionSigner.GetAddr().String()) + return nil + } + if len(s.writeClients) == 1 { // Single client - use direct approach return s.writeClients[0].SendTransaction(ctx, signedTx) @@ -342,19 +376,25 @@ func (s *Sender) SendTransaction(contextID string, target *common.Address, data return common.Hash{}, 0, fmt.Errorf("failed to create signed transaction, err: %w", err) } - // Insert the transaction into the pending transaction table. - // A corner case is that the transaction is inserted into the table but not sent to the chain, because the server is stopped in the middle. - // This case will be handled by the checkPendingTransaction function. - if err = s.pendingTransactionOrm.InsertPendingTransaction(s.ctx, contextID, s.getSenderMeta(), signedTx, blockNumber); err != nil { - log.Error("failed to insert transaction", "from", s.transactionSigner.GetAddr().String(), "nonce", s.transactionSigner.GetNonce(), "err", err) - return common.Hash{}, 0, fmt.Errorf("failed to insert transaction, err: %w", err) + // In dry-run mode, skip pending transaction tracking to avoid polluting the DB. + if !s.config.DryRun { + // Insert the transaction into the pending transaction table. + // A corner case is that the transaction is inserted into the table but not sent to the chain, because the server is stopped in the middle. + // This case will be handled by the checkPendingTransaction function. + if err = s.pendingTransactionOrm.InsertPendingTransaction(s.ctx, contextID, s.getSenderMeta(), signedTx, blockNumber); err != nil { + log.Error("failed to insert transaction", "from", s.transactionSigner.GetAddr().String(), "nonce", s.transactionSigner.GetNonce(), "err", err) + return common.Hash{}, 0, fmt.Errorf("failed to insert transaction, err: %w", err) + } } if err := s.sendTransactionToMultipleClients(signedTx); err != nil { - // Delete the transaction from the pending transaction table if it fails to send. - if updateErr := s.pendingTransactionOrm.DeleteTransactionByTxHash(s.ctx, signedTx.Hash()); updateErr != nil { - log.Error("failed to delete transaction", "tx hash", signedTx.Hash().String(), "from", s.transactionSigner.GetAddr().String(), "nonce", signedTx.Nonce(), "err", updateErr) - return common.Hash{}, 0, fmt.Errorf("failed to delete transaction, err: %w", updateErr) + // In dry-run mode, skip pending transaction cleanup. + if !s.config.DryRun { + // Delete the transaction from the pending transaction table if it fails to send. + if updateErr := s.pendingTransactionOrm.DeleteTransactionByTxHash(s.ctx, signedTx.Hash()); updateErr != nil { + log.Error("failed to delete transaction", "tx hash", signedTx.Hash().String(), "from", s.transactionSigner.GetAddr().String(), "nonce", signedTx.Nonce(), "err", updateErr) + return common.Hash{}, 0, fmt.Errorf("failed to delete transaction, err: %w", updateErr) + } } log.Error("failed to send tx", "tx hash", signedTx.Hash().String(), "from", s.transactionSigner.GetAddr().String(), "nonce", signedTx.Nonce(), "err", err) diff --git a/scroll-contracts b/scroll-contracts index 81f0db72ca..dfbd661520 160000 --- a/scroll-contracts +++ b/scroll-contracts @@ -1 +1 @@ -Subproject commit 81f0db72ca5335e0dddfaa99cb415e3d1cee895f +Subproject commit dfbd661520ac30505a773881728cc5cfb005978b diff --git a/tests/shadow-testing/.env.example b/tests/shadow-testing/.env.example new file mode 100644 index 0000000000..3a44df2cc2 --- /dev/null +++ b/tests/shadow-testing/.env.example @@ -0,0 +1,57 @@ +# Shadow Coordinator + Prover Environment Variables +# Copy this file to .env and fill in real values + +# ============================================================================ +# PRODUCTION RDS (read-only, via IDC port-forward) +# ============================================================================ +PROD_DB_HOST=localhost +PROD_DB_PORT=15432 +PROD_DB_NAME=rollup +PROD_DB_USER=YOUR_PROD_USER_HERE +PROD_DB_PASSWORD=YOUR_PROD_PASSWORD_HERE + +# Full DSN (constructed from above, or override directly) +# PROD_DB=postgresql://YOUR_PROD_USER_HERE:YOUR_PROD_PASSWORD_HERE@localhost:15432/rollup + +# ============================================================================ +# SHADOW DATABASE (local PostgreSQL in Docker) +# ============================================================================ +SHADOW_DB_HOST=localhost +SHADOW_DB_PORT=5433 +SHADOW_DB_NAME=shadow_rollup +SHADOW_DB_USER=postgres +SHADOW_DB_PASSWORD=YOUR_SHADOW_PASSWORD_HERE + +# Full DSN (constructed from above, or override directly) +# SHADOW_DB=postgresql://postgres:YOUR_SHADOW_PASSWORD_HERE@localhost:5433/shadow_rollup + +# ============================================================================ +# COORDINATOR AUTH +# ============================================================================ +# JWT secret for prover login challenge-response. +# MUST match between coordinator config and prover expectations. +COORDINATOR_AUTH_SECRET=YOUR_RANDOM_SECRET_HERE + +# ============================================================================ +# DOCKER IMAGE TAG +# ============================================================================ +IMAGE_TAG=v4.7.13-openvm16 + +# ============================================================================ +# L2 RPC ENDPOINT +# Must support debug_executionWitness and debug_dbGet. +# https://mainnet-rpc.scroll.io works; https://rpc.scroll.io does NOT. +# ============================================================================ +L2_RPC=https://mainnet-rpc.scroll.io + +# ============================================================================ +# VERIFIER ASSETS PATH +# Directory containing subdirectories: openvm-0.5.6, openvm-v0.7.1, openvm-v0.8.0 +# ============================================================================ +VERIFIER_DIR=/tmp/shadow-verifier-assets + +# ============================================================================ +# DATA IMPORT LIMITS +# ============================================================================ +BATCH_LIMIT=50 +BUNDLE_LIMIT=20000 diff --git a/tests/shadow-testing/.gitignore b/tests/shadow-testing/.gitignore new file mode 100644 index 0000000000..1977096c6c --- /dev/null +++ b/tests/shadow-testing/.gitignore @@ -0,0 +1,21 @@ +# Work directory (logs, pid files, generated configs) +.work/ + +# Anvil state files (large, environment-specific) +states/*.json + +# Generated configs (from templates) +.work/*.json + +# Prover work directories +.work/prover-*/ + +# Relayer logs +.work/relayer-*.log + +# Coordinator logs +.work/coordinator-*.log + +# Actual config files with secrets (use .template files instead) +configs/*.json +__pycache__/ diff --git a/tests/shadow-testing/Makefile b/tests/shadow-testing/Makefile new file mode 100644 index 0000000000..44c0ec1fe6 --- /dev/null +++ b/tests/shadow-testing/Makefile @@ -0,0 +1,230 @@ +# Shadow Testing Makefile +# Usage: +# make all CONFIG=mainnet BUNDLE_RANGE=17297:17301 +# make env CONFIG=mainnet BUNDLE_RANGE=17297:17301 +# make prove CONFIG=mainnet BUNDLE_RANGE=17297:17301 +# make finalize CONFIG=mainnet BUNDLE_RANGE=17297:17301 +# make stop + +CONFIG ?= mainnet +BUNDLE_RANGE ?= 17297:17301 +SCRIPT_DIR := $(shell cd "$(dir $(lastword $(MAKEFILE_LIST)))" && pwd) +CONFIG_FILE := $(SCRIPT_DIR)/configs/$(CONFIG).json +WORK_DIR := $(SCRIPT_DIR)/.work + +# Extract values from JSON config (requires jq) +FORK_URL := $(shell jq -r '.fork.url' $(CONFIG_FILE) 2>/dev/null) +FORK_BLOCK := $(shell jq -r '.fork.block_number' $(CONFIG_FILE) 2>/dev/null) +ANVIL_RPC := $(shell jq -r '.fork.anvil_rpc' $(CONFIG_FILE) 2>/dev/null) +DB_DSN := $(shell jq -r '.db.dsn' $(CONFIG_FILE) 2>/dev/null) +GENESIS := $(shell jq -r '.genesis' $(CONFIG_FILE) 2>/dev/null) +SCROLL_CHAIN := $(shell jq -r '.contracts.scroll_chain' $(CONFIG_FILE) 2>/dev/null) +L1_MSG_QUEUE := $(shell jq -r '.contracts.l1_message_queue_v2' $(CONFIG_FILE) 2>/dev/null) +ROLLUP_VERIF := $(shell jq -r '.contracts.rollup_verifier' $(CONFIG_FILE) 2>/dev/null) +DEPLOYED_VERIF:= $(shell jq -r '.contracts.deployed_verifier' $(CONFIG_FILE) 2>/dev/null) +OWNER := $(shell jq -r '.contracts.owner' $(CONFIG_FILE) 2>/dev/null) +PROVER_EOA := $(shell jq -r '.accounts.prover_eoa' $(CONFIG_FILE) 2>/dev/null) +COMMIT_EOA := $(shell jq -r '.accounts.commit_eoa' $(CONFIG_FILE) 2>/dev/null) +LAST_FINALIZED:= $(shell jq -r '.reset.last_finalized_batch_index' $(CONFIG_FILE) 2>/dev/null) +LAST_COMMITTED:= $(shell jq -r '.reset.last_committed_batch_index // empty' $(CONFIG_FILE) 2>/dev/null) +NEXT_QUEUE := $(shell jq -r '.reset.next_unfinalized_queue_index' $(CONFIG_FILE) 2>/dev/null) +CODEC_VERSION := $(shell jq -r '.reset.codec_version' $(CONFIG_FILE) 2>/dev/null) + +# ─── Validation ────────────────────────────────────────────────────────────── +.PHONY: check-deps + +check-deps: + @command -v jq >/dev/null 2>&1 || { echo "jq is required"; exit 1; } + @command -v cast >/dev/null 2>&1 || { echo "cast (Foundry) is required"; exit 1; } + @test -f $(CONFIG_FILE) || { echo "Config file not found: $(CONFIG_FILE)"; exit 1; } + +# ─── Environment Setup ─────────────────────────────────────────────────────── + +env: check-deps anvil-up db-reset + @echo "" + @echo "✅ Environment ready for $(CONFIG)" + @echo " Anvil RPC: $(ANVIL_RPC)" + @echo " Bundle range: $(BUNDLE_RANGE)" + +anvil-up: check-deps + @echo "🚀 Setting up Anvil fork ($(CONFIG))..." + $(SCRIPT_DIR)/scripts/01-setup-anvil.sh \ + --fork-url "$(FORK_URL)" \ + --fork-block "$(FORK_BLOCK)" \ + --anvil-rpc "$(ANVIL_RPC)" \ + --state-file "$(WORK_DIR)/anvil-$(CONFIG).state.json" \ + --last-finalized "$(LAST_FINALIZED)" \ + --last-committed "$(LAST_COMMITTED)" \ + --next-queue "$(NEXT_QUEUE)" \ + --deployed-verifier "$(DEPLOYED_VERIF)" \ + --prover-eoa "$(PROVER_EOA)" \ + --commit-eoa "$(COMMIT_EOA)" \ + --owner "$(OWNER)" \ + --scroll-chain "$(SCROLL_CHAIN)" \ + --l1-msg-queue "$(L1_MSG_QUEUE)" \ + --rollup-verifier "$(ROLLUP_VERIF)" + +db-reset: check-deps + @echo "🗄️ Preparing shadow DB..." + $(SCRIPT_DIR)/scripts/02-prepare-db.sh \ + --db-dsn "$(DB_DSN)" \ + --bundle-range "$(BUNDLE_RANGE)" + +# ─── Proving Phase ──────────────────────────────────────────────────────────── +.PHONY: prove prover-up wait-proofs + +prove: prover-up wait-proofs + +prover-up: + @echo "🔬 Starting provers..." + $(SCRIPT_DIR)/scripts/04-prover-up.sh \ + --config "$(CONFIG)" \ + --bundle-range "$(BUNDLE_RANGE)" + +wait-proofs: + @echo "⏳ Waiting for proofs to complete..." + $(SCRIPT_DIR)/scripts/05-wait-for-proofs.sh \ + --db-dsn "$(DB_DSN)" \ + --bundle-range "$(BUNDLE_RANGE)" + +# ─── Finalization Phase ────────────────────────────────────────────────────── +.PHONY: finalize relayer-up wait-finalize + +finalize: relayer-up wait-finalize + +relayer-up: + @echo "📤 Starting relayer..." + $(SCRIPT_DIR)/scripts/06-run-relayer.sh \ + --config "$(CONFIG)" + +wait-finalize: + @echo "⏳ Waiting for finalization..." + $(SCRIPT_DIR)/scripts/07-wait-for-finalize.sh \ + --anvil-rpc "$(ANVIL_RPC)" \ + --scroll-chain "$(SCROLL_CHAIN)" \ + --bundle-range "$(BUNDLE_RANGE)" + +# ─── Sepolia-specific Pipeline ─────────────────────────────────────────────── +.PHONY: sepolia-all + +sepolia-all: env prove finalize + @echo "" + @echo "🎉 Sepolia shadow test complete!" + @echo " Bundles: $(BUNDLE_RANGE)" + @echo " Anvil RPC: $(ANVIL_RPC)" + +# ─── Full Pipeline ──────────────────────────────────────────────────────────── +.PHONY: all + +all: env prove finalize + @echo "" + @echo "🎉 Shadow test complete for $(CONFIG)!" + @echo " Bundles: $(BUNDLE_RANGE)" + @echo " Anvil RPC: $(ANVIL_RPC)" + +# ─── Status & Verification ─────────────────────────────────────────────────── +.PHONY: status verify + +status: check-deps + @echo "📊 Current status:" + @echo "" + @echo "Anvil lastFinalizedBatchIndex:" + @cast call "$(SCROLL_CHAIN)" "lastFinalizedBatchIndex()(uint256)" --rpc-url "$(ANVIL_RPC)" 2>/dev/null || echo " (Anvil not running)" + @echo "" + @echo "Bundle proving status:" + @psql "$(DB_DSN)" -c "SELECT index, proving_status, rollup_status FROM bundle WHERE index BETWEEN $(subst :, AND ,$(BUNDLE_RANGE)) ORDER BY index;" 2>/dev/null || echo " (DB not accessible)" + +verify: check-deps + @echo "🔍 Verifying on-chain finalization..." + $(SCRIPT_DIR)/scripts/07-wait-for-finalize.sh \ + --anvil-rpc "$(ANVIL_RPC)" \ + --scroll-chain "$(SCROLL_CHAIN)" \ + --bundle-range "$(BUNDLE_RANGE)" \ + --verify-only + +# ─── Docker-Orchestrated Pipeline ────────────────────────────────────────────── +.PHONY: docker-all docker-env docker-prove docker-finalize docker-stop + +docker-all: + @echo "🐳 Docker-orchestrated full pipeline..." + $(SCRIPT_DIR)/scripts/08-docker-orchestrate.sh \ + --config $(CONFIG) \ + --bundle-range $(BUNDLE_RANGE) \ + --phase all + +docker-env: + @echo "🐳 Docker environment setup..." + $(SCRIPT_DIR)/scripts/08-docker-orchestrate.sh \ + --config $(CONFIG) \ + --bundle-range $(BUNDLE_RANGE) \ + --phase env + +docker-prove: + @echo "🐳 Docker proving phase..." + $(SCRIPT_DIR)/scripts/08-docker-orchestrate.sh \ + --config $(CONFIG) \ + --bundle-range $(BUNDLE_RANGE) \ + --phase prove + +docker-finalize: + @echo "🐳 Docker finalization phase..." + $(SCRIPT_DIR)/scripts/08-docker-orchestrate.sh \ + --config $(CONFIG) \ + --bundle-range $(BUNDLE_RANGE) \ + --phase finalize + +docker-stop: + @echo "🛑 Stopping Docker services..." + -docker compose -f $(SCRIPT_DIR)/docker-compose.yml down 2>/dev/null || true + @if [ -f $(WORK_DIR)/anvil-$(CONFIG).pid ]; then \ + kill $$(cat $(WORK_DIR)/anvil-$(CONFIG).pid) 2>/dev/null || true; \ + rm -f $(WORK_DIR)/anvil-$(CONFIG).pid; \ + fi + @echo "Done." + +# ─── Cleanup ───────────────────────────────────────────────────────────────── +.PHONY: stop clean + +stop: docker-stop + @echo "🛑 Stopping bare-metal services..." + @if [ -f $(WORK_DIR)/anvil.pid ]; then \ + kill $$(cat $(WORK_DIR)/anvil.pid) 2>/dev/null || true; \ + rm -f $(WORK_DIR)/anvil.pid; \ + fi + @echo "Done." + +clean: stop + @echo "🧹 Cleaning up..." + rm -rf $(WORK_DIR)/* + @echo "Done." + +# ─── Help ──────────────────────────────────────────────────────────────────── +.PHONY: help + +help: + @echo "Shadow Testing Makefile" + @echo "" + @echo "Docker Targets (recommended):" + @echo " make docker-all CONFIG= BUNDLE_RANGE= Full docker pipeline" + @echo " make docker-env CONFIG= BUNDLE_RANGE= Setup env (postgres + anvil + import)" + @echo " make docker-prove CONFIG= BUNDLE_RANGE= Prove (coordinator + prover)" + @echo " make docker-finalize CONFIG= BUNDLE_RANGE= Finalize (relayer)" + @echo " make docker-stop Stop docker services" + @echo "" + @echo "Bare-Metal Targets:" + @echo " make all CONFIG= BUNDLE_RANGE= Full pipeline" + @echo " make env CONFIG= BUNDLE_RANGE= Setup Anvil + DB" + @echo " make prove CONFIG= BUNDLE_RANGE= Prove bundles" + @echo " make finalize CONFIG= BUNDLE_RANGE= Finalize bundles" + @echo " make status CONFIG= BUNDLE_RANGE= Check status" + @echo " make verify CONFIG= BUNDLE_RANGE= Verify on-chain" + @echo " make stop Stop all services" + @echo " make clean Stop + cleanup" + @echo "" + @echo "Configs: $(shell ls $(SCRIPT_DIR)/configs/*.json 2>/dev/null | xargs -n1 basename | sed 's/.json//g' | xargs)" + @echo "" + @echo "Example (docker):" + @echo " make docker-all CONFIG=mainnet BUNDLE_RANGE=17302:17305" + @echo "" + @echo "Example (bare-metal):" + @echo " make all CONFIG=mainnet BUNDLE_RANGE=17297:17301" diff --git a/tests/shadow-testing/README.md b/tests/shadow-testing/README.md new file mode 100644 index 0000000000..8668f050aa --- /dev/null +++ b/tests/shadow-testing/README.md @@ -0,0 +1,78 @@ +# Shadow Testing Toolkit + +One-command toolkit for running Scroll shadow fork tests against Anvil. + +## Quick Start + +### Docker (Recommended) + +```bash +cd tests/shadow-testing +make docker-all CONFIG=mainnet BUNDLE_RANGE=17302:17305 +``` + +### Bare-Metal + +```bash +cd tests/shadow-testing +make all CONFIG=mainnet BUNDLE_RANGE=17297:17301 +``` + +See `make help` for all targets. + +## Documentation + +| Document | What It Covers | +|----------|----------------| +| [`docs/GUIDE.md`](docs/GUIDE.md) | Full setup guide — step-by-step manual setup, architecture, configuration | +| [`docs/TROUBLESHOOTING.md`](docs/TROUBLESHOOTING.md) | Structured pitfalls, traps, and agent checklists | +| [`docs/LESSONS_LEARNED.md`](docs/LESSONS_LEARNED.md) | Time-lined debugging stories from past shadow tests | +| [`docs/CONTRACTS.md`](docs/contract-addresses.md) | L1 contract addresses per network | + +## Directory Structure + +``` +configs/ # JSON config templates (copy and edit) +scripts/ # Numbered pipeline scripts (01-setup-anvil.sh …) +docs/ # Documentation +states/ # Anvil state files (gitignored) +.work/ # Runtime logs, pid files (gitignored) +``` + +## Prerequisites + +- [Foundry](https://book.getfoundry.sh/) (`cast`, `forge`) +- `jq` +- `docker compose` (for Docker mode) +- PostgreSQL client (`psql`) +- Access to production RDS (via IDC port-forward) + +## Configurations + +Copy templates and fill in secrets: + +```bash +cp configs/mainnet.json.template configs/mainnet.json +cp configs/sepolia.json.template configs/sepolia.json +cp configs/coordinator.json.template configs/coordinator.json +# Edit the files and replace placeholders: +# - YOUR_ALCHEMY_API_KEY (in mainnet.json / sepolia.json fork.url) +# - YOUR_SHADOW_DB_PASSWORD (in mainnet.json / sepolia.json db.dsn) +``` + +## How It Works + +The pipeline has three phases: + +1. **Environment Setup** (`make env` or `make docker-env`) + Start Anvil fork, import bundle data from production RDS, reset status. + +2. **Proving** (`make prove` or `make docker-prove`) + Start coordinator + provers, wait for proofs to complete. + +3. **Finalization** (`make finalize` or `make docker-finalize`) + Start relayer, wait for on-chain finalization. + +## Contributing + +When you discover a new trap or workaround, add it to `docs/TROUBLESHOOTING.md` (structured) or `docs/LESSONS_LEARNED.md` (narrative). diff --git a/tests/shadow-testing/configs/coordinator.json.template b/tests/shadow-testing/configs/coordinator.json.template new file mode 100644 index 0000000000..2d6f356250 --- /dev/null +++ b/tests/shadow-testing/configs/coordinator.json.template @@ -0,0 +1,40 @@ +{ + "prover_manager": { + "provers_per_session": 1, + "session_attempts": 5, + "external_prover_threshold": 32, + "bundle_collection_time_sec": 180, + "batch_collection_time_sec": 180, + "chunk_collection_time_sec": 3600, + "verifier": { + "min_prover_version": "v4.4.45", + "verifiers": [ + { + "assets_path": "assets", + "fork_name": "galileoV2" + } + ] + } + }, + "db": { + "driver_name": "postgres", + "dsn": "postgres://postgres:YOUR_SHADOW_DB_PASSWORD@shadow-postgres:5432/shadow_rollup?sslmode=disable", + "maxOpenNum": 200, + "maxIdleNum": 20 + }, + "l2": { + "validium_mode": false, + "chain_id": 534352, + "l2geth": { + "endpoint": "https://l2geth-rpc-proxy.mainnet.aws.scroll.io" + } + }, + "auth": { + "secret": "YOUR_COORDINATOR_AUTH_SECRET", + "challenge_expire_duration_sec": 3600, + "login_expire_duration_sec": 3600 + }, + "sequencer": { + "decryption_key": "" + } +} diff --git a/tests/shadow-testing/configs/mainnet.json.template b/tests/shadow-testing/configs/mainnet.json.template new file mode 100644 index 0000000000..b984743d43 --- /dev/null +++ b/tests/shadow-testing/configs/mainnet.json.template @@ -0,0 +1,45 @@ +{ + "name": "mainnet-shadow-fork", + "fork": { + "url": "https://eth-mainnet.g.alchemy.com/v2/YOUR_ALCHEMY_API_KEY", + "block_number": 25202217, + "anvil_rpc": "http://localhost:18545" + }, + "contracts": { + "scroll_chain": "0xa13BAF47339d63B743e7Da8741db5456DAc1E556", + "l1_message_queue_v2": "0x56971da63A3C0205184FEF096E9ddFc7A8C2D18a", + "rollup_verifier": "0x4CEA3E866e7c57fD75CB0CA3E9F5f1151D4Ead3F", + "deployed_verifier": "0xb1F2C5c1ea2885278a1070350d12d3D8824265B0", + "owner": "0x798576400F7D662961BA15C6b3F3d813447a26a6" + }, + "accounts": { + "prover_eoa": "0x410E7FD80a3Fc1E62A4D3450d11b71b812006eB9", + "commit_eoa": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266" + }, + "reset": { + "last_finalized_batch_index": 517765, + "next_unfinalized_queue_index": 0, + "codec_version": 10, + "last_committed_batch_index": 517819 + }, + "db": { + "dsn": "postgresql://postgres:YOUR_SHADOW_DB_PASSWORD@localhost:5433/shadow_rollup" + }, + "genesis": "tests/prover-e2e/mainnet-galileoV2/genesis.json", + "assets": { + "assets_v2": "coordinator/build/bin/assets_v2" + }, + "prover": { + "name_prefix": "galileo6-shadowfork-prover", + "circuit_version": "v0.13.1", + "s3_base_url": "https://circuit-release.s3.us-west-2.amazonaws.com/scroll-zkvm/galileov2/" + }, + "relayer": { + "validium_mode": false, + "min_codec_version": 7, + "chain_monitor_enabled": false + }, + "e2e": { + "l2_rpc": "https://l2geth-rpc-proxy.mainnet.aws.scroll.io" + } +} diff --git a/tests/shadow-testing/configs/relayer.json.template b/tests/shadow-testing/configs/relayer.json.template new file mode 100644 index 0000000000..6d84879cc3 --- /dev/null +++ b/tests/shadow-testing/configs/relayer.json.template @@ -0,0 +1,75 @@ +{ + "l2_config": { + "endpoint": "{{L2_ENDPOINT}}", + "relayer_config": { + "sender_config": { + "endpoint": "{{ANVIL_RPC}}", + "check_balance": false, + "dry_run": false, + "tx_type": "DynamicFeeTx", + "escalate_blocks": 10, + "escalate_multiple_num": 11, + "escalate_multiple_den": 10, + "confirmations": "0x0", + "max_gas_price": 10000000000000, + "max_blob_gas_price": 10000000000000, + "min_gas_tip": 0, + "check_pending_time": 60, + "max_pending_blob_txs": 3, + "fusaka_timestamp": 9999999999 + }, + "commit_sender_signer_config": { + "signer_type": "PrivateKey", + "private_key_signer_config": { + "private_key": "{{COMMIT_KEY}}" + } + }, + "finalize_sender_signer_config": { + "signer_type": "PrivateKey", + "private_key_signer_config": { + "private_key": "{{FINALIZE_KEY}}" + } + }, + "rollup_contract_address": "{{SCROLL_CHAIN}}", + "batch_submission": { + "min_batches": 1, + "max_batches": 1, + "timeout": 7200, + "backlog_max": 200, + "blob_fee_tolerance": 500000000 + }, + "gas_oracle": { + "enabled": false + }, + "chain_monitor": { + "enabled": {{CHAIN_MONITOR_ENABLED}} + }, + "batch_committer": { + "enable_test_env_bypass_features": true + }, + "validium_mode": {{VALIDIUM_MODE}} + }, + "chunk_proposer_config": { + "propose_interval_milliseconds": 10000, + "max_l2_gas_per_chunk": 100000000, + "chunk_timeout_sec": 900, + "max_uncompressed_batch_bytes_size": 63488 + }, + "batch_proposer_config": { + "propose_interval_milliseconds": 10000, + "batch_timeout_sec": 1800, + "max_chunks_per_batch": 15, + "max_uncompressed_batch_bytes_size": 129024 + }, + "bundle_proposer_config": { + "max_batch_num_per_bundle": 30, + "bundle_timeout_sec": 3600 + }, + "blob_uploader_config": { + "start_batch": 0 + } + }, + "db_config": { + "dsn": "{{DB_DSN}}" + } +} diff --git a/tests/shadow-testing/configs/sepolia.json.template b/tests/shadow-testing/configs/sepolia.json.template new file mode 100644 index 0000000000..27b5535550 --- /dev/null +++ b/tests/shadow-testing/configs/sepolia.json.template @@ -0,0 +1,51 @@ +{ + "name": "sepolia-shadow-fork", + "fork": { + "url": "https://eth-sepolia.g.alchemy.com/v2/YOUR_ALCHEMY_API_KEY", + "block_number": 10878976, + "anvil_rpc": "http://localhost:18546" + }, + "contracts": { + "scroll_chain": "0x2D567EcE699Eabe5afCd141eDB7A4f2D0D6ce8a0", + "l1_message_queue_v2": "0xA0673eC0A48aa924f067F1274EcD281A10c5f19F", + "rollup_verifier": "0x8A360c7F6fca548507017DdeD732bFe7E078F963", + "deployed_verifier": "", + "owner": "0xbE57544Eaf3515E888614a464EC9e0ad38f73e37" + }, + "accounts": { + "prover_eoa": "0x410E7FD80a3Fc1E62A4D3450d11b71b812006eB9", + "commit_eoa": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266" + }, + "reset": { + "last_finalized_batch_index": 127915, + "next_unfinalized_queue_index": 0, + "codec_version": 10 + }, + "db": { + "dsn": "postgresql://YOUR_SHADOW_DB_USER:YOUR_SHADOW_DB_PASSWORD@localhost:5442/sepolia_scroll" + }, + "genesis": "tests/prover-e2e/sepolia-galileoV2/genesis.json", + "assets": { + "assets_v2": "coordinator/build/bin/assets_v2" + }, + "prover": { + "name_prefix": "sepolia-shadowfork-prover", + "circuit_version": "v0.13.1", + "s3_base_url": "https://circuit-release.s3.us-west-2.amazonaws.com/scroll-zkvm/galileov2/" + }, + "coordinator": { + "l2geth": "https://l2geth-rpc-proxy.sepolia.aws.scroll.io" + }, + "relayer": { + "validium_mode": false, + "min_codec_version": 7, + "chain_monitor_enabled": false + }, + "e2e": { + "begin_block": 17086000, + "end_block": 17086005, + "l2_rpc": "https://sepolia-rpc.scroll.io", + "fork_name": "galileoV2", + "zkvm_version": "v0.8.0" + } +} diff --git a/tests/shadow-testing/docker-compose.yml b/tests/shadow-testing/docker-compose.yml new file mode 100644 index 0000000000..87bcedc5fc --- /dev/null +++ b/tests/shadow-testing/docker-compose.yml @@ -0,0 +1,213 @@ +version: "3.8" + +services: + # ─── PostgreSQL (Shadow DB) ──────────────────────────────────────────────── + postgres: + image: postgres:15-alpine + container_name: shadow-postgres + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: shadow_pass + POSTGRES_DB: shadow_rollup + ports: + - "5433:5432" + volumes: + - postgres-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 5s + timeout: 5s + retries: 5 + networks: + - shadow-net + + # ─── Coordinator API ─────────────────────────────────────────────────────── + coordinator: + image: scrolltech/coordinator-api:e2e-test + container_name: shadow-coordinator + profiles: ["coordinator"] + command: + - "--config" + - "/app/conf/config.json" + - "--http" + - "--http.addr" + - "0.0.0.0" + - "--http.port" + - "8390" + ports: + - "8390:8390" + volumes: + - ./configs/coordinator.json:/app/conf/config.json:ro + - ../../coordinator/build/bin/assets_v2:/app/assets:ro + - ../../tests/prover-e2e/mainnet-galileoV2/genesis.json:/app/conf/genesis.json:ro + depends_on: + postgres: + condition: service_healthy + networks: + - shadow-net + restart: unless-stopped + + # ─── Relayer ─────────────────────────────────────────────────────────────── + relayer: + image: ubuntu:22.04 + container_name: shadow-relayer + profiles: ["relayer"] + entrypoint: ["/app/rollup_relayer"] + command: + - "--config" + - "/app/config.json" + - "--genesis" + - "/app/conf/genesis.json" + - "--min-codec-version" + - "7" + - "--verbosity" + - "3" + volumes: + - ./.work/relayer-${CONFIG:-mainnet}.json:/app/config.json:ro + - ../../tests/prover-e2e/mainnet-galileoV2/genesis.json:/app/conf/genesis.json:ro + - ../../rollup/build/bin/rollup_relayer:/app/rollup_relayer:ro + depends_on: + - postgres + networks: + - shadow-net + restart: unless-stopped + + # ─── Anvil (optional — usually started by script for fork flexibility) ───── + anvil: + image: ghcr.io/foundry-rs/foundry:latest + container_name: shadow-anvil + profiles: ["anvil"] + entrypoint: ["anvil"] + command: + - "--fork-url" + - "${FORK_URL:-https://eth-mainnet.g.alchemy.com/v2/demo}" + - "--fork-block-number" + - "${FORK_BLOCK:-25202217}" + - "--block-time" + - "12" + - "--port" + - "8545" + - "--state" + - "/data/anvil.state.json" + ports: + - "18545:8545" + volumes: + - ./states:/data + networks: + - shadow-net + restart: unless-stopped + + # ─── Prover GPU-0 ────────────────────────────────────────────────────────── + prover-gpu-0: + image: scrolltech/prover:${PROVER_VERSION:-e2e-test} + container_name: shadow-prover-gpu-0 + profiles: ["prover"] + ports: + - "10080:10080" + environment: + - RUST_MIN_STACK=16777216 + - CUDA_VISIBLE_DEVICES=0 + volumes: + - ./.work/prover-0.json:/prover/conf/config.json:ro + - ./.work/prover-0:/prover/.work + - ~/.openvm/params:/root/.openvm/params:ro + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: ['0'] + capabilities: [gpu] + depends_on: + - coordinator + networks: + - shadow-net + restart: unless-stopped + + # ─── Prover GPU-1 ────────────────────────────────────────────────────────── + prover-gpu-1: + image: scrolltech/prover:${PROVER_VERSION:-e2e-test} + container_name: shadow-prover-gpu-1 + profiles: ["prover"] + ports: + - "10081:10080" + environment: + - RUST_MIN_STACK=16777216 + - CUDA_VISIBLE_DEVICES=0 + volumes: + - ./.work/prover-1.json:/prover/conf/config.json:ro + - ./.work/prover-1:/prover/.work + - ~/.openvm/params:/root/.openvm/params:ro + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: ['1'] + capabilities: [gpu] + depends_on: + - coordinator + networks: + - shadow-net + restart: unless-stopped + + # ─── Prover GPU-2 ────────────────────────────────────────────────────────── + prover-gpu-2: + image: scrolltech/prover:${PROVER_VERSION:-e2e-test} + container_name: shadow-prover-gpu-2 + profiles: ["prover"] + ports: + - "10082:10080" + environment: + - RUST_MIN_STACK=16777216 + - CUDA_VISIBLE_DEVICES=0 + volumes: + - ./.work/prover-2.json:/prover/conf/config.json:ro + - ./.work/prover-2:/prover/.work + - ~/.openvm/params:/root/.openvm/params:ro + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: ['2'] + capabilities: [gpu] + depends_on: + - coordinator + networks: + - shadow-net + restart: unless-stopped + + # ─── Prover GPU-3 ────────────────────────────────────────────────────────── + prover-gpu-3: + image: scrolltech/prover:${PROVER_VERSION:-e2e-test} + container_name: shadow-prover-gpu-3 + profiles: ["prover"] + ports: + - "10083:10080" + environment: + - RUST_MIN_STACK=16777216 + - CUDA_VISIBLE_DEVICES=0 + volumes: + - ./.work/prover-3.json:/prover/conf/config.json:ro + - ./.work/prover-3:/prover/.work + - ~/.openvm/params:/root/.openvm/params:ro + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: ['3'] + capabilities: [gpu] + depends_on: + - coordinator + networks: + - shadow-net + restart: unless-stopped + +volumes: + postgres-data: + +networks: + shadow-net: + driver: bridge diff --git a/tests/shadow-testing/docs/GUIDE.md b/tests/shadow-testing/docs/GUIDE.md new file mode 100644 index 0000000000..3dc207cf9d --- /dev/null +++ b/tests/shadow-testing/docs/GUIDE.md @@ -0,0 +1,1008 @@ +# Shadow Coordinator + Prover Testing Guide + +This guide documents how to set up a **shadow coordinator** + **local prover** environment for testing proof generation without interfering with production. This approach is significantly simpler than a full shadow fork — we use a local coordinator with imported production task data and a local prover that fetches tasks from it. + +## Architecture + +``` +┌──────────────────┐ ┌──────────────────┐ ┌──────────────────┐ +│ Production RDS │ │ Shadow DB │ │ Shadow │ +│ (read-only via │────▶│ (local :5433) │────▶│ Coordinator │ +│ port-forward) │ │ │ │ (localhost:8390)│ +└──────────────────┘ └──────────────────┘ └────────┬─────────┘ + │ + │ assigns tasks + ▼ +┌──────────────────┐ ┌──────────────────┐ ┌──────────────────┐ +│ L2 RPC │ │ Local Prover │ │ Verifier Assets │ +│ (mainnet-rpc. │◀────│ (GPU/CPU) │ │ (/tmp/shadow- │ +│ scroll.io) │ │ │ │ verifier-assets)│ +└──────────────────┘ └──────────────────┘ └──────────────────┘ +``` + +## Prerequisites + +### Hardware +- GPU with CUDA support (tested on RTX 3090) +- ~50GB disk space for Docker images + verifier assets + circuit downloads +- 16GB+ RAM + +### Software +- Docker + docker-compose +- PostgreSQL client (`psql`) +- Rust toolchain (for local prover binary) +- `kubectl` or SSH access to IDC for port-forwarding to production RDS + +### Network +- Access to IDC machine with port-forward to mainnet RDS (e.g., `idc-us-1-19`) +- Internet access for L2 RPC and S3 circuit downloads + +## Quick Start + +If you just want to get running, use the provided script: + +```bash +# 1. Set up shadow PostgreSQL +cd tests/shadow-testing +./setup.sh --postgres + +# 2. Import production task data (requires RDS port-forward) +./import-production-data.sh + +# 3. Start shadow coordinator +./setup.sh --coordinator + +# 4. Start prover (in another terminal) +./setup.sh --prover +``` + +## Step-by-Step Setup + +### Step 1: Set up IDC Port-Forward to Production RDS + +On the IDC machine (e.g., `idc-us-1-19`), ensure the port-forward is active: + +```bash +# Mainnet RDS should be accessible on localhost:15432 +# Credentials are loaded from .env (see .env.example) +psql -h localhost -p 15432 -U "$PROD_DB_USER" -d rollup -c "SELECT version();" +``` + +If not already set up, configure SSH tunnel or kubectl port-forward from your workstation. + +### Step 2: Start Local PostgreSQL (Shadow DB) + +```bash +docker run -d \ + --name shadow-coordinator-postgres \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD="${SHADOW_DB_PASSWORD}" \ + -e POSTGRES_DB=shadow_rollup \ + -p 5433:5432 \ + -v shadow-coordinator-postgres-data:/var/lib/postgresql/data \ + postgres:15 + +# Wait for DB to be ready +sleep 5 +docker exec shadow-coordinator-postgres pg_isready -U postgres +``` + +### Step 3: Download Verifier Assets + +The coordinator needs verifier assets for each supported fork: + +```bash +VERIFIER_DIR="/tmp/shadow-verifier-assets" +mkdir -p "$VERIFIER_DIR" + +# feynman (OpenVM 0.5.6) +mkdir -p "$VERIFIER_DIR/openvm-0.5.6" +# Download or copy verifier assets for feynman + +# galileo (v0.7.1) +mkdir -p "$VERIFIER_DIR/openvm-v0.7.1" +# Download or copy verifier assets for galileo + +# galileoV2 (v0.8.0) — NOTE: v0.8.0 does NOT use /releases/ prefix in S3 URLs +mkdir -p "$VERIFIER_DIR/openvm-v0.8.0" +# Download or copy verifier assets for galileoV2 +``` + +> ⚠️ **Important**: v0.8.0 assets use `v0.8.0/` path prefix, NOT `releases/v0.8.0/`. Using the wrong prefix causes HTTP 403 errors. + +### Step 4: Initialize Shadow DB Schema + +Use the coordinator's built-in migration or apply schema manually. The coordinator container will auto-migrate on first start. + +### Step 5: Import Production Task Data + +Export the latest N batches + their chunks + bundles from production RDS and import into shadow DB: + +```bash +# Edit these variables as needed +# Credentials loaded from .env (see tests/shadow-testing/.env.example) +PROD_DB="postgresql://${PROD_DB_USER}:${PROD_DB_PASSWORD}@${PROD_DB_HOST}:${PROD_DB_PORT}/${PROD_DB_NAME}" +SHADOW_DB="postgresql://${SHADOW_DB_USER}:${SHADOW_DB_PASSWORD}@${SHADOW_DB_HOST}:${SHADOW_DB_PORT}/${SHADOW_DB_NAME}" +BATCH_LIMIT=50 + +# Export batches +psql "$PROD_DB" -c " + COPY ( + SELECT * FROM batch + ORDER BY index DESC + LIMIT $BATCH_LIMIT + ) TO STDOUT WITH CSV HEADER; +" > /tmp/batches.csv + +# Export chunks in those batches +psql "$PROD_DB" -c " + COPY ( + SELECT c.* FROM chunk c + JOIN batch b ON b.start_chunk_index <= c.index AND c.index <= b.end_chunk_index + WHERE b.index IN (SELECT index FROM batch ORDER BY index DESC LIMIT $BATCH_LIMIT) + ORDER BY c.index + ) TO STDOUT WITH CSV HEADER; +" > /tmp/chunks.csv + +# Export bundles (all or limited) +psql "$PROD_DB" -c " + COPY ( + SELECT * FROM bundle + ORDER BY index DESC + LIMIT 20000 + ) TO STDOUT WITH CSV HEADER; +" > /tmp/bundles.csv + +# Import into shadow DB (truncate first) +psql "$SHADOW_DB" -c "TRUNCATE batch, chunk, bundle CASCADE;" + +# Use \copy for local import +psql "$SHADOW_DB" -c "\\copy batch FROM '/tmp/batches.csv' WITH CSV HEADER;" +psql "$SHADOW_DB" -c "\\copy chunk FROM '/tmp/chunks.csv' WITH CSV HEADER;" +psql "$SHADOW_DB" -c "\\copy bundle FROM '/tmp/bundles.csv' WITH CSV HEADER;" + +# Reset proving status to unassigned (1) +psql "$SHADOW_DB" -c "UPDATE chunk SET proving_status = 1, total_attempts = 0, active_attempts = 0;" +psql "$SHADOW_DB" -c "UPDATE batch SET proving_status = 1, total_attempts = 0, active_attempts = 0, chunk_proofs_status = 0;" +psql "$SHADOW_DB" -c "UPDATE bundle SET proving_status = 1, total_attempts = 0, active_attempts = 0;" +``` + +### Step 6: Populate l2_block Table + +The coordinator needs `l2_block` records to format chunk tasks (for block hashes and hardfork name resolution). + +Use the provided Python script or fetch blocks via L2 RPC: + +```bash +python3 tests/shadow-testing/scripts/fetch-l2-blocks.py \ + --rpc https://mainnet-rpc.scroll.io \ + --db "postgresql://$SHADOW_DB_USER:$SHADOW_DB_PASSWORD@$SHADOW_DB_HOST:$SHADOW_DB_PORT/$SHADOW_DB_NAME" \ + --start-block 26000000 \ + --end-block 27000000 +``` + +After inserting blocks, link them to chunks: + +```bash +psql "$SHADOW_DB" -c " + UPDATE l2_block lb + SET chunk_hash = c.hash + FROM chunk c + WHERE lb.number >= c.start_block_number + AND lb.number <= c.end_block_number; +" +``` + +### Step 7: Start Shadow Coordinator + +Use Docker (recommended) or run locally: + +```bash +# Via Docker +docker run -d \ + --name shadow-coordinator-api-test \ + --network host \ + -v /tmp/shadow-coordinator-config.json:/app/conf/config.json \ + -v /tmp/shadow-verifier-assets:/verifier \ + zhuoatscroll/coordinator-api:v4.7.13-openvm16 + +# Wait for startup (takes 2-3 min for OpenVM keygen) +docker logs -f shadow-coordinator-api-test | grep -m1 "Start coordinator api successfully" +``` + +### Step 8: Start Prover + +Build or use prebuilt binary: + +```bash +# Build locally +cd /path/to/scroll-repo +cargo build --release -p prover-bin + +# Or use Docker image +docker run -d \ + --name shadow-prover \ + --network host \ + --gpus all \ + -v /tmp/prover-local.json:/app/config.json \ + -v ~/.openvm/params:/root/.openvm/params:ro \ + zhuoatscroll/prover:v4.7.13-openvm16 + +# Or run binary directly +./target/release/prover --config /tmp/prover-local.json +``` + +> ℹ️ **Note**: Prover will download circuit assets from S3 on first run (several GB). Subsequent runs use cached assets in `.work/galileo/`. + +## Monitoring + +### Check coordinator health +```bash +curl -s http://localhost:8390/ | head +``` + +### Check prover health +```bash +curl -s http://localhost:10080/health +``` + +### Watch coordinator logs +```bash +docker logs -f shadow-coordinator-api-test --tail 100 +``` + +### Watch prover logs +```bash +# If running via docker +docker logs -f shadow-prover --tail 100 + +# If running binary directly, logs go to stdout +``` + +### Check DB task status +```bash +psql "$SHADOW_DB" -c " + SELECT proving_status, COUNT(*) FROM chunk GROUP BY proving_status; +" +``` + +Proving status values: +- `1` = Unassigned +- `2` = Assigned +- `3` = Proving +- `4` = Proven (success) +- `5` = Failed + +## Troubleshooting + +### Coordinator says "Start coordinator api successfully" but prover gets no tasks +- Verify `l2_block` table has records for the chunk's block range +- Check `proving_status = 1` on chunks +- Check `codec_version != 5` (chunks with codec_version = 5 are skipped) +- Ensure chunk's `end_block_number <= coordinator's block height` + +### "mismatched post-state root" or codec errors +- Verify you're using blocks after the hardfork. For GalileoV2 (codec V10), use blocks ≥ 33,750,000 on mainnet. +- Ensure `SCROLL_FORK_NAME` and verifier assets match the block's fork. + +### "Failed to execute witness" or "Method not found" +- The L2 RPC must support `debug_executionWitness` and `debug_dbGet`. +- `https://mainnet-rpc.scroll.io` supports these; `https://rpc.scroll.io` does not. + +### "Failed to get l1 messages in block" (-32601) +- Your RPC does not support `scroll_getL1MessagesInBlock`. This is non-fatal if the block contains no L1 messages. +- If L1 messages exist, you need an RPC that supports this method. + +### S3 403 errors when downloading circuit assets +- v0.8.0 assets: `https://circuit-release.s3.us-west-2.amazonaws.com/scroll-zkvm/v0.8.0/` +- v0.7.1 and earlier: `https://circuit-release.s3.us-west-2.amazonaws.com/scroll-zkvm/releases/v0.7.1/` +- Verify with `curl -sI ` before running. + +### "bind: address already in use" (port 8390) +- Kill old coordinator: `pkill -f coordinator_api` or `docker rm -f shadow-coordinator-api-test` + +### Port conflicts with local PostgreSQL +- If you have system PostgreSQL on 5432, use 5433 for shadow DB (already configured). +- Ensure all configs use the correct port. + +### Multi-GPU prover cache conflicts +When running multiple prover instances on the same machine, the shared `.work/galileo` cache directory can cause `File exists (os error 17)` conflicts if two provers write the same temp file simultaneously. + +**Mitigation**: Ensure each prover has its own work directory, or symlink `.work/galileo` to a shared read-only cache while giving each instance a distinct write directory. Example launch script: +```bash +for i in 0 1 2 3; do + mkdir -p /tmp/prover-gpu${i}/work + ln -s /shared/cache/galileo /tmp/prover-gpu${i}/work/galileo + CUDA_VISIBLE_DEVICES=$i ./prover --config /tmp/prover-gpu${i}/config.json & +done +``` + +### Bundle proving never starts +If coordinator is actively assigning chunk/batch tasks but never assigns bundle tasks, the most likely cause is **orphan bundles** — bundle records whose corresponding batch data no longer exists in the shadow DB. + +**Diagnosis**: +```sql +-- Count bundles with no linked batches +SELECT COUNT(*) FROM bundle b +WHERE NOT EXISTS ( + SELECT 1 FROM batch bat + WHERE bat.index BETWEEN b.start_batch_index AND b.end_batch_index +); +``` + +**Root cause**: The bundle table often retains historical records from production (e.g., batch 308516+) while the batch table only holds recently imported batches (e.g., 517760+). Coordinator's `GetUnassignedBundle` picks the lowest-index bundle with `batch_proofs_status = 2`, finds it has no batches, and fails silently in a loop. + +**Fix**: +```sql +UPDATE bundle +SET batch_proofs_status = 1 +WHERE index NOT IN ( + SELECT DISTINCT b.index + FROM bundle b + JOIN batch bat ON bat.index BETWEEN b.start_batch_index AND b.end_batch_index +); +``` + +### DB data inconsistency after import +If imported chunks have `proving_status = 2` (assigned) but `proof = NULL`, coordinator may incorrectly set `batch.chunk_proofs_status = 2` and then fail when formatting batch tasks. + +**Fix**: +```sql +UPDATE chunk SET proving_status = 1, total_attempts = 0, active_attempts = 0 +WHERE proving_status = 2 AND proof IS NULL; + +UPDATE batch SET chunk_proofs_status = 0 +WHERE chunk_proofs_status != 0 + AND EXISTS ( + SELECT 1 FROM chunk c + WHERE c.batch_hash = batch.hash AND c.proving_status != 4 + ); +``` + +## Configuration Reference + +### Shadow Coordinator Config + +See `configs/shadow-coordinator-config.json` in this directory. + +Key fields: +- `db.dsn`: Points to shadow PostgreSQL +- `l2.l2geth.endpoint`: L2 RPC with `debug_executionWitness` support +- `prover_manager.verifier.verifiers`: List of verifier asset paths and fork names + +### Prover Config + +See `configs/prover-local.json` in this directory. + +Key fields: +- `sdk_config.coordinator.base_url`: Shadow coordinator API (`http://localhost:8390`) +- `circuits.galileoV2.base_url`: S3 path for circuit assets (no `/releases/` for v0.8.0) +- `sdk_config.prover.supported_proof_types`: `[1, 2, 3]` for chunk, batch, bundle + +## Rollup Relayer Dry-Run Mode + +For testing the **rollup-relayer's transaction construction logic** (e.g., `finalizeBundle` calldata) without spending real gas or modifying chain state, the sender module supports a **dry-run mode**. + +When `"dry_run": true` is set in the sender config: +- Transactions are **simulated** via `eth_call` instead of being broadcast +- `pending_transaction` table is **not** populated (avoids DB pollution) +- Nonce is still incremented to simulate real behavior +- If the `eth_call` fails (e.g., contract revert), the error is propagated just like a real send failure + +### Usage + +1. Build the rollup-relayer binary: +```bash +cd rollup && go build -o rollup_relayer ./cmd/rollup_relayer/app +``` + +2. Configure `dry_run: true` in the sender config (see `tests/shadow-testing/configs/rollup-relayer-dryrun.json`) + +3. Start the relayer: +```bash +./rollup_relayer --config /path/to/rollup-relayer-dryrun.json +``` + +### What Dry-Run Verifies + +| Aspect | Verified? | Notes | +|--------|-----------|-------| +| Calldata encoding (ABI pack) | ✅ | `constructFinalizeBundlePayloadCodecV7` etc. | +| Gas estimation | ✅ | Full `EstimateGas` + `CreateAccessList` path | +| Contract revert | ✅ | `eth_call` returns revert reason | +| Signature / nonce | ⚠️ | Nonce incremented but tx not broadcast | +| Pending tx lifecycle | ❌ | Skipped to avoid DB pollution | +| Receipt confirmation | ❌ | No real tx = no receipt | + +For **full end-to-end** validation (including signature + receipt), use **Anvil** with `evm_snapshot`/`evm_revert` instead. + +### Anvil + Mock ScrollChain Setup (Recommended for Dry-Run) + +For the most realistic dry-run testing, deploy a minimal mock ScrollChain contract on a local Anvil node: + +```bash +# 1. Start Anvil forked from mainnet (or standalone) +anvil --fork-url https://eth-mainnet.g.alchemy.com/v2/YOUR_KEY --fork-block-number 33878313 + +# 2. Deploy mock contract (minimal Solidity with no-op commitBatches / finalizeBundle) +cat > MockScrollChain.sol << 'EOF' +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; +contract MockScrollChain { + mapping(address => bool) public isProver; + address public owner; + constructor() { owner = msg.sender; } + function addProver(address _prover) external { + require(msg.sender == owner, "Not owner"); + isProver[_prover] = true; + } + function commitBatches(uint8 version, bytes32 parentBatchHash, bytes32 batchHash) external {} + function finalizeBundlePostEuclidV2NoProof(bytes calldata, uint256, bytes32, bytes32) external {} + function finalizeBundlePostEuclidV2(bytes calldata, uint256, bytes32, bytes32, bytes calldata) external {} +} +EOF + +# Compile and deploy +solc --bin MockScrollChain.sol -o /tmp/mock +BYTECODE=$(cat /tmp/mock/MockScrollChain.bin) +cast send --rpc-url http://localhost:18545 \ + --private-key 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 \ + --create "0x$BYTECODE" +# → contractAddress: 0x1fA02b2d6A771842690194Cf62D91bdd92BfE28d + +# 3. Fund sender accounts and add prover +COMMIT_ADDR="0x1e32ABcfE6db15c1570709E3fC02725335f50A47" +FINALIZE_ADDR="0x33e0F539E31B35170FAaA062af703b76a8282bf7" +cast rpc anvil_setBalance "$COMMIT_ADDR" "0x3635c9adc5dea00000" --rpc-url http://localhost:18545 +cast rpc anvil_setBalance "$FINALIZE_ADDR" "0x3635c9adc5dea00000" --rpc-url http://localhost:18545 +cast send "addProver(address)" "$FINALIZE_ADDR" --rpc-url http://localhost:18545 \ + --private-key 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 +``` + +**Key sender config changes**: +```json +{ + "sender_config": { + "endpoint": "http://localhost:18545", + "dry_run": true + } +} +``` + +**Dry-run gas estimation skip**: Anvil may fail `EstimateGas` on blob transactions or missing functions. A small patch to `rollup/internal/controller/sender/estimategas.go` skips gas estimation in dry-run mode: +```go +func (s *Sender) estimateGasLimit(...) (uint64, *types.AccessList, error) { + if s.config.DryRun { + return 10000000, nil, nil // skip estimation + } + // ... original logic +} +``` + +### What We Verified in Practice + +| Transaction | Status | Notes | +|-------------|--------|-------| +| `commitBatches` | ✅ `eth_call` succeeded | Selector `0x9bbaa2ba` via mock `commitBatches(uint8,bytes32,bytes32)` | +| `finalizeBundlePostEuclidV2NoProof` | ✅ `eth_call` succeeded | Selector `0xbd6f916b` via mock no-op | +| `finalizeBundlePostEuclidV2` (with proof) | ✅ `eth_call` succeeded | Bundle 17301 with valid `OpenVMBundleProof` | + +### ⚠️ Critical Discovery: Anvil Must Fork Ethereum Mainnet, NOT Scroll Mainnet + +When querying `0xa13BAF47339d63B743e7Da8741db5456DAc1E556` on **Scroll L2** (`scroll-mainnet.g.alchemy.com`), the contract appears to have no ScrollChain functions and an empty implementation slot. This led to confusion — the address seemed to be a ProxyAdmin rather than the ScrollChain proxy. + +**The root cause**: We were querying the **wrong chain**. The ScrollChain proxy `0xa13B...` is deployed on **Ethereum L1**, not Scroll L2. When queried on Ethereum mainnet: + +- **Implementation**: `0x0a20703878e68e587c59204cc0ea86098b8c3ba7` (ScrollChain logic) +- **Admin**: `0xEB803eb3F501998126bf37bB823646Ed3D59d072` (ProxyAdmin) +- **Functions verified**: `lastFinalizedBatchIndex()`, `committedBatches(uint256)`, `isSequencer(address)`, `isProver(address)`, `commitBatches(uint8,bytes32,bytes32)`, `finalizeBundlePostEuclidV2(bytes,uint256,bytes32,bytes32,bytes)` + +### Real ScrollChain Proxy Dry-Run Testing + +For testing against the **actual deployed ScrollChain contract** on an Anvil fork: + +```bash +# 1. Start Anvil forked from ETHEREUM mainnet (NOT Scroll mainnet) +anvil --fork-url https://eth-mainnet.g.alchemy.com/v2/YOUR_KEY \ + --fork-block-number 25206000 \ + --port 18545 \ + --no-rate-limit \ + --block-time 5 + +# 2. Run takeover script (impersonate owner, add sequencer/prover) +# See scroll-devnets/charts/shadow-fork/rollup-relayer/scripts/takeover-l1-contracts.sh +# Key addresses: +# L1_SCROLL_CHAIN_PROXY_ADDR=0xa13BAF47339d63B743e7Da8741db5456DAc1E556 +# L1_SCROLL_OWNER_ADDR=0x798576400F7D662961BA15C6b3F3d813447a26a6 +# FORKED_L1_SCROLL_OWNER_ADDR=0x909D2900A1Ec2B518EAFe11811Da0c1Fc8729a73 +# FORKED_L1_SCROLL_OWNER_PRIVATE_KEY=0x93d9b2e68479131dfa877a77cef8a286986940ab2de677a4790d17267462dd5e + +# 3. Set balances for relayer senders +COMMIT_ADDR="0x1e32ABcfE6db15c1570709E3fC02725335f50A47" +FINALIZE_ADDR="0x33e0F539E31B35170FAaA062af703b76a8282bf7" +cast rpc anvil_setBalance "$COMMIT_ADDR" "0x21e19e0c9bab2400000" --rpc-url http://localhost:18545 +cast rpc anvil_setBalance "$FINALIZE_ADDR" "0x21e19e0c9bab2400000" --rpc-url http://localhost:18545 + +# 4. Configure relayer to use REAL proxy address +# In config: "rollup_contract_address": "0xa13BAF47339d63B743e7Da8741db5456DAc1E556" +``` + +**Important**: If blob base fee is extremely high on the forked block (causing `Insufficient funds`), mine empty blocks to reduce `excessBlobGas`: +```bash +cast rpc anvil_mine 400 --rpc-url http://localhost:18545 +``` + +### Dry-Run Results with Real ScrollChain Proxy + +| Transaction | Status | Notes | +|-------------|--------|-------| +| `commitBatches` | ⚠️ `eth_call` reached contract | Reverted with `ErrorIncorrectBatchHash()` — shadow DB batch data is ahead of fork block state | +| `finalizeBundlePostEuclidV2` | ✅ **Succeeded** | Bundle 17330 (batch 517809) finalized successfully with real mainnet proof. See "End-to-End finalizeBundlePostEuclidV2 Dry-Run Success" below. | + +**Why `commitBatches` reverts**: The shadow DB contains batches 518565+ but the Anvil fork block (25206000) only has batches committed up to ~517816. The parent batch hash in the calldata doesn't match what the contract expects, triggering `ErrorIncorrectBatchHash()`. + +This is **expected and actually confirms the pipeline works** — the relayer is successfully constructing and sending calldata to the real ScrollChain implementation, and the contract's validation logic is executing correctly. + +For `finalizeBundlePostEuclidV2`, the batch was already committed on mainnet at the fork block, so no `commitBatches` call is needed — we only need the proof and verifier to match. + +--- + +## Real Verifier Deployment + +### Critical Discovery: Deployed Verifier Digests Must Match Mainnet Proof + +We initially deployed a fresh `ZkEvmVerifierPostFeynman` (address `0xc323...`) using S3 digest files. However, **direct verification failed** with `VerificationFailed (0x439cc0cd)` because the VK digests in our deployed verifier did **not** match the digests embedded in the mainnet proof. + +The mainnet proof for bundle 17330 was generated with VK digests: +- `verifierDigest1`: `0x0091609acb607118f47f756c0f4db9aad227420326cbda96f0303384e0bbf8e3` +- `verifierDigest2`: `0x009305f0762291e3cdd805ff6d6e81f1d135dbfdeb3ecf30ad82c3855dde7909` + +Our deployed verifier had completely different digests (`0x00398b...` / `0x002178...`). + +**Recommended Solution**: Copy the exact mainnet verifier contract code (including embedded immutables) to Anvil using `anvil_setCode`: + +```bash +# Copy mainnet ZkEvmVerifierPostFeynman wrapper (0x0dE1...) +MAINNET_VERIFIER="0x0dE180164Dc571522457101F5c47B2eaB36d0A82" +CODE=$(cast code $MAINNET_VERIFIER --rpc-url https://ethereum-rpc.publicnode.com) +cast rpc anvil_setCode $MAINNET_VERIFIER $CODE --rpc-url http://localhost:18545 + +# Copy its Plonk verifier (0x749f...) +PLONK="0x749fc77a1a131632a8b88e8703e489557660c75e" +PLONK_CODE=$(cast code $PLONK --rpc-url https://ethereum-rpc.publicnode.com) +cast rpc anvil_setCode $PLONK $PLONK_CODE --rpc-url http://localhost:18545 +``` + +This preserves the exact immutables (plonkVerifier address, digests, protocolVersion) from mainnet. + +### Alternative: Extract Digests from the Proof Itself + +If you must deploy a fresh verifier (e.g., testing a new circuit version), **do not use S3 `digest_1.hex` / `digest_2.hex`**. These files often return 403 or contain digests that do not match the specific proof you are testing. + +Instead, extract digests directly from the proof's `instances` array: + +```python +import base64, json + +# proof_json is the decoded proof from coordinator/bundle table +proof_data = base64.b64decode(proof_json['proof']['proof']) +instances_data = base64.b64decode(proof_json['proof']['instances']) + +# instances are 12 × 32-byte Fr elements = 384 bytes, followed by app commits +# verifierDigest1 = app_exe_commit = instances_bytes[384:416] (big-endian) +# verifierDigest2 = app_vm_commit = instances_bytes[416:448] (big-endian) +digest1 = '0x' + instances_data[384:416].hex() +digest2 = '0x' + instances_data[416:448].hex() +``` + +For the v0.8.0 circuit used to prove bundle 17330: +- `digest1` = `0x00398b786b500ca759ca2de2aee9c73bd8e28f1c80b49e1c53bc060a9a649269` +- `digest2` = `0x0021785a05e931b447c8d6463f4547f92081a92ee357af26e1c6f6ecfe373d67` + +Then deploy with: +```bash +forge create --broadcast --evm-version cancun --rpc-url http://localhost:18545 \ + src/libraries/verifier/ZkEvmVerifierPostFeynman.sol:ZkEvmVerifierPostFeynman \ + --constructor-args +``` + +### Register Copied Verifier + +```bash +MVRV="0x4cea3e866e7c57fd75cb0ca3e9f5f1151d4ead3f" +OWNER="0x909d2900a1ec2b518eafe11811da0c1fc8729a73" +ANVIL_VERIFIER="0x0dE180164Dc571522457101F5c47B2eaB36d0A82" + +# Impersonate owner and register +cast rpc anvil_impersonateAccount $OWNER --rpc-url http://localhost:18545 +cast send $MVRV \ + "updateVerifier(uint256,uint64,address)" \ + 10 0 $ANVIL_VERIFIER \ + --from $OWNER --rpc-url http://localhost:18545 --unlocked +``` + +> **Note**: `latestVerifier[10]` returns a struct; use `getVerifier(10, batchIndex)` to confirm routing. + +### Previous Deployment Attempt (Incorrect Digests) + +For reference, the initially deployed verifier (wrong digests) was: + +```bash +# Step 1: Deploy Plonk Verifier +VER="v0.8.0" +BASE_URL="https://circuit-release.s3.us-west-2.amazonaws.com/scroll-zkvm/$VER" +curl -sL -o /tmp/verifier.bin "$BASE_URL/verifier/verifier.bin" +PLONK_BYTECODE=$(xxd -p /tmp/verifier.bin | tr -d '\n') +cast send --rpc-url http://localhost:18545 --chain 1 \ + --from 0x909D2900A1Ec2B518EAFe11811Da0c1Fc8729a73 --unlocked \ + --create "$PLONK_BYTECODE" +# → Plonk Verifier: 0xe1c0b68e8377deee8eff9267e00981a45f2967e2 + +# Step 2: Deploy ZkEvmVerifierPostFeynman +cd scroll-contracts && git checkout 42de954bee237cfa478a5b443ac0aeb900aca5ad +DIGEST1=$(curl -s "$BASE_URL/bundle/digest_1.hex") +DIGEST2=$(curl -s "$BASE_URL/bundle/digest_2.hex") +forge create --broadcast --evm-version cancun --rpc-url http://localhost:18545 \ + --from 0x909D2900A1Ec2B518EAFe11811Da0c1Fc8729a73 --unlocked \ + src/libraries/verifier/ZkEvmVerifierPostFeynman.sol:ZkEvmVerifierPostFeynman \ + --constructor-args 0xe1c0b68e8377deee8eff9267e00981a45f2967e2 0x$DIGEST1 0x$DIGEST2 8 +# → ZkEvmVerifierPostFeynman: 0xc3230A4C89a5Ce0455414215e533de4D8849b3f8 +``` + +This verifier **does not work** with mainnet bundle proofs because the S3 digest files do not match the VK used for the specific bundle being tested. Always use `anvil_setCode` to copy the mainnet verifier instead. + +### Critical Discovery: Anvil `eth_call` vs `anvil_setStorageAt` + +**Refined conclusion** (updated after further testing): + +- `anvil_setStorageAt` on **mapping slots** (e.g., `committedBatches[batchIndex]`) is visible to `eth_getStorageAt` but is **cached and ignored** by `eth_call` / `eth_sendTransaction` during contract execution. This is an Anvil bug. +- `anvil_setStorageAt` on **direct variable slots** (e.g., `miscData` at slot 161, `nextUnfinalizedQueueIndex` at slot 104) **does work** and is visible to `eth_call`. + +**Implications**: +- You **can** override simple state variables like `lastFinalizedBatchIndex`, `nextUnfinalizedQueueIndex`, etc. +- You **cannot** override mapping entries like `committedBatches[517809]` or `finalizedStateRoots[517808]`. +- For mappings, either fork at a block where the desired state already exists, or use a mock contract. + +### Deployed Contract Addresses (Anvil Fork) + +| Contract | Address | Notes | +|----------|---------|-------| +| ScrollChain Proxy | `0xa13BAF47339d63B743e7Da8741db5456DAc1E556` | Forked from mainnet | +| MultipleVersionRollupVerifier | `0x4CEA3E866e7c57fD75CB0CA3E9F5f1151D4Ead3F` | Forked from mainnet | +| **ZkEvmVerifierPostFeynman (v10)** | `0x0dE180164Dc571522457101F5c47B2eaB36d0A82` | **Copied from mainnet** ✅ | +| Plonk Verifier (v10) | `0x749fc77a1a131632a8b88e8703e489557660c75e` | Copied from mainnet | +| ZkEvmVerifierPostFeynman (wrong) | `0xc3230A4C89a5Ce0455414215e533de4D8849b3f8` | Deployed with S3 digests — **do not use** | + +--- + +## End-to-End finalizeBundlePostEuclidV2 Dry-Run Success + +We successfully executed `finalizeBundlePostEuclidV2` end-to-end on Anvil using **real mainnet proof data** from shadow DB bundle 17330. + +### Bundle 17330 Parameters + +| Field | Value | +|-------|-------| +| Bundle index | 17330 | +| Batch index | 517809 | +| Codec version | 10 (GalileoV2) | +| Num batches | 1 | +| `postStateRoot` | `0x28ff638e237ad6a0f2eebaab84f254dd4fca8a16297413c29fcd70f8b1b3fd85` | +| `withdrawRoot` | `0xe88d24e9153438c91f94c32026cb49730212f32ac4652367b07c71f96ce063d9` | +| `batchHash` | `0xeadeee9af865c6d13df6b66a45b3f3f161e6211aeb7d86e075a645f0e6a58f9e` | +| `prevStateRoot` | `0x4d21a5ca662bffc2d650a4d24a445617c3eb7159a28b13548ec5421a3ba08ee7` | +| `prevBatchHash` | `0xd6d7d027ef32d393a4aff7b04c1577bcb1f7fdc44834797e48f7e01581615a58` | +| `totalL1MessagesPoppedOverall` | 998288 | +| `msgQueueHash` | `0x5b08e5befde15d3acbf1a3e0e99622a6ac3fa62049cdfa62ba984ab700000000` | +| Mainnet finalize tx | `0x753f8f9ca01d4e67f710c6dab8ce0b17a17a7ad46a9d7480d92657803a36ca24` | + +### Public Input Verification + +The 204-byte public input is constructed as: + +``` +chain_id(8) || msg_queue_hash(32) || num_batches(4) || prev_state_root(32) || prev_batch_hash(32) || post_state_root(32) || batch_hash(32) || withdraw_root(32) +``` + +The `ZkEvmVerifierPostFeynman` contract prepends `protocolVersion = 10` (32 bytes) and computes: + +```solidity +publicInputHash = keccak256(abi.encodePacked(protocolVersion, publicInput)) +``` + +- `protocolVersion` = 10 (GalileoV2) +- `publicInput` = 204 bytes (standard EuclidV2 format) +- Actual input to keccak256 = **236 bytes** (32-byte version prefix + 204-byte public input) + +Computed hash: `0xcd4421bad526bd108d9ae8c2af3d46ea1a986207f0b8c1af781b601c1ae50e5a` + +This **exactly matches** `bundle_pi_hash` from the proof metadata. + +#### Bundle Proof Format + +The `aggrProof` passed to `finalizeBundlePostEuclidV2` is: + +``` +bundleProof = instances[:384] + proof_bytes = 1760 bytes total +├── 384 bytes = accumulator (12 Fr elements) +└── 1376 bytes = Plonk proof +``` + +The `ZkEvmVerifierPostFeynman.verify()` function inserts `digest1`, `digest2`, and `publicInputHash` expansion into the calldata before forwarding to the Plonk verifier. + +### Pre-Execution Setup Required + +Because the Anvil fork block (25213457) is **after** the real finalization block (25198501), several state variables had already advanced past the values needed for the dry-run. We applied the following fixes: + +#### 1. Add Authorized Prover + +The prover authorization was lost after `anvil_reset`. Re-add: + +```bash +SCROLL_CHAIN="0xa13BAF47339d63B743e7Da8741db5456DAc1E556" +OWNER="0x798576400F7D662961BA15C6b3F3d813447a26a6" +PROVER="0xc48DfbcdC4ef4cdACFf94eE7385020b7a7CE195f" + +cast rpc anvil_setBalance $OWNER 0x56bc75e2d63100000 --rpc-url http://localhost:18545 +cast send $SCROLL_CHAIN "addProver(address)" $PROVER \ + --from $OWNER --rpc-url http://localhost:18545 --unlocked +``` + +> ⚠️ **Anvil Default Account Is Not an EOA**: Anvil's default test account `0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266` has contract code (`0xef0100...`) in fork mode. `ScrollChain.addProver()` checks `_account.code.length == 0` and will revert with `ErrorAccountIsNotEOA`. Always use a freshly generated EOA (e.g., from `cast wallet new`) as the prover address. +> +> If the real owner account is an EOA with delegation code (e.g., ERC-7702), you may need to temporarily swap the proxy owner via `anvil_setStorageAt` on slot 51, call `addProver`, then restore the original owner. + +#### 2. Override `lastFinalizedBatchIndex` + +Set `miscData` (slot 161) so `lastFinalizedBatchIndex = 517808`: + +```bash +cast rpc anvil_setStorageAt $SCROLL_CHAIN 0xa1 \ + 0x0000000000000000000000016a1bb977000000000007e6b0000000000007e6d3 \ + --rpc-url http://localhost:18545 +``` + +> Layout: `lastCommitted(8) | lastFinalized(8) | lastFinalizeTimestamp(4) | flags(1) | reserved(7)` + +#### 3. Override `L1MessageQueueV2.nextUnfinalizedQueueIndex` + +Set slot 104 to `0` (the finalize call will update it to 998288): + +```bash +MQV2="0x56971da63A3C0205184FEF096E9ddFc7A8C2D18a" +cast rpc anvil_setStorageAt $MQV2 0x68 0x0 --rpc-url http://localhost:18545 +``` + +#### 4. Copy Mainnet Verifier + +See "Real Verifier Deployment" above for the `anvil_setCode` commands to copy the mainnet verifier wrapper and its Plonk verifier. + +### Execution + +```bash +# Extract proof from mainnet finalize transaction +python3 << 'PYEOF' +import subprocess, json +result = subprocess.run([ + 'cast', 'tx', '0x753f8f9ca01d4e67f710c6dab8ce0b17a17a7ad46a9d7480d92657803a36ca24', + '--json', '--rpc-url', 'https://ethereum-rpc.publicnode.com' +], capture_output=True, text=True) +tx = json.loads(result.stdout) +input_hex = tx['input'] +data = bytes.fromhex(input_hex[2:]) +# ... decode batchHeader, totalL1MessagesPoppedOverall, postStateRoot, withdrawRoot, aggrProof +PYEOF + +# Send transaction +SCROLL_CHAIN="0xa13BAF47339d63B743e7Da8741db5456DAc1E556" +PROVER="0xc48DfbcdC4ef4cdACFf94eE7385020b7a7CE195f" + +cast rpc anvil_setBalance $PROVER 0x56bc75e2d63100000 --rpc-url http://localhost:18545 +cast send $SCROLL_CHAIN --from $PROVER $(cat /tmp/finalize_calldata.hex) \ + --rpc-url http://localhost:18545 --unlocked +``` + +### Result + +- **Transaction Hash**: `0x0000ba738dbcc27e89db8e545532cdc125a9d50c42683032d92ed30203ea8d65` +- **Status**: Success ✅ +- **Gas Used**: 425,719 +- **Block**: 25213522 + +### Post-Execution State + +| Variable | Value | +|----------|-------| +| `lastFinalizedBatchIndex` | 517809 | +| `finalizedStateRoots[517809]` | `0x28ff638e237ad6a0f2eebaab84f254dd4fca8a16297413c29fcd70f8b1b3fd85` | +| `L1MessageQueueV2.nextUnfinalizedQueueIndex` | 998288 | + +### Key Takeaways + +1. **Always copy the mainnet verifier** — Deploying a new verifier with S3 digests will fail because the digests may not match the specific proof being tested. If you must deploy fresh, extract digests from `proof.instances[12]` and `proof.instances[13]`. +2. **`anvil_setStorageAt` works for direct variables** but not for mapping entries. Use it for `miscData`, `nextUnfinalizedQueueIndex`, etc. +3. **Fork block matters** — If the fork block is after the real finalization, you must manually reset `lastFinalizedBatchIndex` and `nextUnfinalizedQueueIndex`. +4. **Public input hash must match exactly** — Any discrepancy in `msg_queue_hash`, `chain_id`, `num_batches`, or roots will cause `VerificationFailed`. +5. **Anvil default account is not an EOA in fork mode** — Use a freshly generated EOA for `addProver`; `0xf39F...` has contract code and will fail the EOA check. +6. **Reset `rollup_status` before relayer finalize** — The shadow DB retains mainnet rollup state (`RollupFinalized` = 5). The relayer's `GetFirstPendingBundle` only queries `rollup_status = RollupPending` (1). You must reset both `bundle` and `batch` tables before the relayer will pick up bundles for finalization. + +## Multi-Bundle Relayer Finalize Test (5 Bundles) + +This test demonstrates running the actual `rollup_relayer` binary against an Anvil mainnet fork to finalize **5 consecutive bundles** (17297–17301, batches 517761–517765) using shadow proofs. + +### Prerequisites + +- Anvil fork running with `lastFinalizedBatchIndex` reset to `517760` +- Shadow proofs generated for all 5 bundles (`proving_status = 4`) +- Verifier `0xb1F2C5c1ea2885278a1070350d12d3D8824265B0` registered as `latestVerifier[10]` +- Prover/finalize EOA `0x410E...` authorized on `ScrollChain` + +### Step 1: Reset DB Rollup Status + +The shadow DB retains mainnet rollup state. Before the relayer can pick up bundles, reset their status: + +```sql +UPDATE bundle SET rollup_status = 1 WHERE index BETWEEN 17297 AND 17301; +UPDATE batch SET rollup_status = 1 WHERE index BETWEEN 517761 AND 517765; +``` + +(`1` = `RollupPending`; without this, `GetFirstPendingBundle` returns nothing.) + +### Step 2: Build and Configure Relayer + +```bash +cd rollup +go build -o /tmp/rollup_relayer ./cmd/rollup_relayer +``` + +Create `/tmp/rollup-relayer-anvil.json`: + +```json +{ + "l2_config": { + "l2_geth": { "endpoint": "https://mainnet-galileo.scroll.io/l2" }, + "relayer_config": { + "sender_config": { + "endpoint": "http://localhost:18545", + "check_balance": false, + "dry_run": false + }, + "commit_sender_signer_config": { + "private_key": "0xac09..." + }, + "finalize_sender_signer_config": { + "private_key": "0x01f1..." + }, + "rollup_contract_address": "0xa13BAF47339d63B743e7Da8741db5456DAc1E556", + "chain_monitor": { "enabled": false }, + "gas_oracle": { "enabled": false }, + "batch_committer": { + "enable_test_env_bypass_features": true + }, + "validium_mode": false + } + }, + "db_config": { + "dsn": "postgresql://postgres:shadow_pass@localhost:5433/shadow_rollup" + } +} +``` + +**Important**: `commit_sender` and `finalize_sender` must be **different addresses**. The relayer enforces this at startup. + +### Step 3: Launch Relayer + +```bash +/tmp/rollup_relayer \ + --config /tmp/rollup-relayer-anvil.json \ + --genesis /home/scroll/zzhang/scroll/tests/prover-e2e/mainnet-galileoV2/genesis.json \ + --min-codec-version 7 \ + --verbosity 3 \ + 2>&1 | tee /tmp/relayer.log +``` + +The relayer starts all modules (L2 watcher, proposers, batch committer, bundle finalizer). The batch committer will fail with `ErrorCallerIsNotSequencer` (expected — the commit sender is not a sequencer), but the **bundle finalizer runs independently every 15 seconds** and will pick up the pending bundles. + +### Step 4: Monitor Finalization + +Watch `/tmp/relayer.log` for: + +``` +{"msg":"Start to roll up zk proof","index":17297,...} +{"msg":"finalizeBundle in layer1","index":17297,"tx hash":"0x6d62...","with proof":"true"} +``` + +### Results + +| Bundle | Batch | Transaction Hash | Status | Gas Used | +|--------|-------|------------------|--------|----------| +| 17297 | 517761 | `0x6d6264...cdaa725` | ✅ Success | 439,987 | +| 17298 | 517762 | `0x071268...1136516` | ✅ Success | 407,455 | +| 17299 | 517763 | `0x8f8894...6cabd5` | ✅ Success | 407,479 | +| 17300 | 517764 | `0xa87721...302cd3` | ✅ Success | 407,419 | +| 17301 | 517765 | `0x41ee42...c9cf89` | ✅ Success | 401,404 | + +**Final `lastFinalizedBatchIndex`**: `517765` (was `517760`) + +All 5 bundles finalized consecutively without manual intervention. Each bundle proof was verified on-chain by the `ZkEvmVerifierPostFeynman` contract deployed at `0xb1F2C5c1ea2885278a1070350d12d3D8824265B0`. + +### Key Differences from CLI Approach + +| Aspect | CLI (`cast send`) | Relayer | +|--------|-------------------|---------| +| Calldata construction | Manual Python script | Relayer reads from DB + constructs automatically | +| Sender management | Single EOA | Separate commit/finalize senders | +| Batch status tracking | None | Updates `bundle` and `batch` `rollup_status` in DB | +| Error handling | Manual retry | Built-in retry and status polling | +| Multi-bundle support | One at a time | Processes all pending bundles automatically | + +## Known Limitations + +1. **L1 messages**: If chunks contain L1 messages, the prover needs `scroll_getL1MessagesInBlock` RPC support. Most public RPCs don't expose this. Workaround: select chunks/blocks with no L1 messages, or use an internal RPC. In non-validium mode, the prover does not call this RPC at all. + +2. **Full batch proving**: Batch tasks require `chunk_proofs_status = 2` (all chunks proven). For quick chunk-only testing, you don't need to prove full batches. + +3. **Coordinator startup time**: First startup performs OpenVM keygen (~2-3 min). Be patient. + +4. **Circuit download**: First prover run downloads ~5-10GB of circuit assets. Ensure good internet. + +5. **Bundle vs batch count mismatch**: The shadow DB's `bundle` table may contain 10,000+ historical records while `batch` only holds ~500 recent ones. This is expected when importing production data — the bundle table retains full history but batches are truncated. **Crucially**, orphan bundles (those with no matching batches) must have `batch_proofs_status = 1` or coordinator will deadlock trying to prove them. See "Bundle proving never starts" in Troubleshooting. + +6. **`finalizeBundlePostEuclidV2` and multi-batch bundles**: The contract computes `numBatches = batchIndex - lastFinalizedBatchIndex`. The proof's `num_batches` must exactly match this value. Single-batch bundles (e.g., bundle 17330 = batch 517809) are the easiest to test because `numBatches = 1`. Multi-batch bundles also work as long as `lastFinalizedBatchIndex` is set so that `batchIndex - lastFinalizedBatchIndex` equals the proof's `num_batches`. + +7. **Local E2E proofs cannot be used on mainnet fork**: Local E2E proofs are generated against a different chain state (genesis batch, different state roots, different message queue). Even if you deploy matching verifier digests, the public input (state roots, batch hashes, message queue hash) will not match the forked mainnet contract state, causing `VerificationFailed`. + +## Automated DB Replication from Mainnet RDS + +The `~/.pgpass` file on this machine contains valid credentials for the mainnet RDS read-only replica: + +```bash +# Verify access +cast psql -h localhost -p 15432 -U mainnet_infra_team_read_only -d mainnet_rollup -c "SELECT COUNT(*) FROM batch;" +# → 517,830 batches +``` + +For automated DB sync, see `scroll-devnets/charts/shadow-fork/rollup-relayer/scripts/copy-db.sh` which uses `postgres-tunnel` to stream data from mainnet RDS to local shadow DB via `COPY ... TO STDOUT | COPY ... FROM STDIN`. + +## Common DB Fixes + +After importing production data or running for extended periods, these SQL fixes resolve common coordinator deadlocks: + +### 1. Reset proving status after import +```sql +UPDATE chunk SET proving_status = 1, total_attempts = 0, active_attempts = 0; +UPDATE batch SET proving_status = 1, total_attempts = 0, active_attempts = 0, chunk_proofs_status = 0; +UPDATE bundle SET proving_status = 1, total_attempts = 0, active_attempts = 0; +``` + +### 2. Mark orphan bundles (no linked batches) +```sql +UPDATE bundle +SET batch_proofs_status = 1 +WHERE index NOT IN ( + SELECT DISTINCT b.index + FROM bundle b + JOIN batch bat ON bat.index BETWEEN b.start_batch_index AND b.end_batch_index +); +``` + +### 3. Fix stale assigned chunks without proofs +```sql +UPDATE chunk SET proving_status = 1, total_attempts = 0, active_attempts = 0 +WHERE proving_status = 2 AND proof IS NULL; + +UPDATE batch SET chunk_proofs_status = 0 +WHERE chunk_proofs_status != 0 + AND EXISTS ( + SELECT 1 FROM chunk c + WHERE c.batch_hash = batch.hash AND c.proving_status != 4 + ); +``` + +## Scripts Reference + +| Script | Purpose | +|--------|---------| +| `setup.sh` | One-command setup for PostgreSQL, coordinator, or prover | +| `import-production-data.sh` | Export from production RDS and import to shadow DB | +| `fetch-l2-blocks.py` | Fetch block headers from L2 RPC and populate `l2_block` table | diff --git a/tests/shadow-testing/docs/LESSONS_LEARNED.md b/tests/shadow-testing/docs/LESSONS_LEARNED.md new file mode 100644 index 0000000000..74e60966e1 --- /dev/null +++ b/tests/shadow-testing/docs/LESSONS_LEARNED.md @@ -0,0 +1,988 @@ +# Shadow Testing Lessons Learned + +## 2026-06-03: "psql timeout" does NOT mean "port is closed" + +### What Happened +1. Ran `psql -h localhost -p 25432 ...` to connect to Sepolia shadow DB. +2. Command timed out after 60s. +3. **Misconclusion**: Assumed port-forward was not established and told user to start SSH tunnel. +4. User asked: "你怎么测试的？telnet $PORT 吗？" +5. Ran `nc -vz localhost 25432` → port was **OPEN**. +6. Re-ran `psql` with correct username → connected instantly. + +### Root Cause +- `psql` timeout can be caused by many things: DNS resolution, SSL handshake failure, wrong username triggering slow auth fallback, etc. +- TCP port being open is a separate layer from application-level connectivity. + +### Rule +> **Always test TCP connectivity first** with `nc`, `telnet`, or `/dev/tcp/host/port` before diagnosing application-level issues. +> Only after confirming the port is open (or closed) should you investigate `psql`-specific parameters. + +### Verification Commands +```bash +# TCP connectivity test (fast, no auth needed) +timeout 3 bash -c 'cat < /dev/null > /dev/tcp/localhost/25432' && echo "Open" || echo "Closed" + +# Or with nc +nc -vz localhost 25432 + +# Then test psql with explicit username +psql -h localhost -p 25432 -U sepolia_infra_user_read_only -d sepolia_scroll -c "SELECT 1;" +``` + +--- + +## 2026-06-03: PostEuclid vs PostFeynman Verifier Mismatch + +### What Happened +1. Deployed a fresh `ZkEvmVerifierPostEuclid` with digests extracted from the shadow proof (`0x00398b78...` / `0x0021785a...`). +2. Updated `MultipleVersionRollupVerifier` to point to the new verifier. +3. `finalizeBundlePostEuclidV2` still reverted with `VerificationFailed()` (`0x439cc0cd`). + +### Root Cause +- `ZkEvmVerifierPostEuclid.computeHash()` does: + ```solidity + bytes32 publicInputHash = keccak256(publicInput); + ``` +- `ZkEvmVerifierPostFeynman.computeHash()` does: + ```solidity + bytes32 publicInputHash = keccak256(abi.encodePacked(protocolVersion, publicInput)); + ``` + where `protocolVersion = (domain << 6) + stf_version = (0 << 6) + 10 = 10`. +- The prover (guest v0.8.0+) computes `bundle_pi_hash` with the `protocolVersion` prefix (`pi_hash_versioned()`). +- Therefore `PostEuclid` always produces a different hash than the proof expects, causing unconditional `VerificationFailed`. + +### How to Prevent This +1. **Always use `ZkEvmVerifierPostFeynman` for guest v0.8.0+ proofs.** +2. **Extract digests from the proof itself**, not from S3 `digest_1.hex` / `digest_2.hex` (those often don't match the specific proof being tested): + ```python + instances = base64.b64decode(proof_json['proof']['instances']) + digest1 = '0x' + instances[384:416].hex() + digest2 = '0x' + instances[416:448].hex() + ``` +3. **Use the provided script** (`scripts/03-deploy-verifier.sh`) which already deploys `PostFeynman` with the correct digests and `protocolVersion = 10`. + +### Recovery Steps +1. Deploy `ZkEvmVerifierPostFeynman` with the same plonk verifier, digest1, digest2, and `protocolVersion = 10`. +2. Call `MultipleVersionRollupVerifier.updateVerifier(10, startBatch, newVerifier)`. +3. Re-test with `cast call` before running the relayer. + +--- + +## 2026-06-03: Relayer nonce desync due to stale pending_transaction table + +### What Happened +1. The relayer successfully sent a `finalizeBundlePostEuclidV2` tx (nonce=1) for bundle 17302. +2. The relayer panicked in the batch committer before the tx could be confirmed. +3. On restart, the relayer initialized nonce from `pending_transaction` (`maxDbNonce=5`), so it used nonce=6. +4. Anvil's on-chain nonce for the finalize sender was still 1 (only nonce=0 had been used for a manual tx). +5. The relayer sent a tx with nonce=6, which got stuck in Anvil's mempool as a "future" tx and was never mined. + +### Root Cause +- `pending_transaction` retained entries from previous relayer runs that were never confirmed (status=1 or 3). +- The relayer's `reset nonce` logic uses `maxDbNonce + 1`, not the actual on-chain nonce. +- When Anvil is restarted or txs are dropped, the DB nonce history becomes stale. + +### How to Prevent This +1. **Before restarting the relayer after a crash or Anvil restart**, clear the finalize sender's pending transactions: + ```sql + DELETE FROM pending_transaction WHERE sender_address = ''; + ``` +2. **Verify nonce consistency on relayer startup**: + ```bash + cast nonce --rpc-url http://localhost:18545 + ``` + This should match `nextDbNonce` in the relayer startup logs. +3. **If they don't match**, stop the relayer, clear `pending_transaction`, and restart. + +### Recovery Steps +1. Stop the relayer. +2. ```sql + DELETE FROM pending_transaction WHERE sender_address = '0x410E7FD80a3Fc1E62A4D3450d11b71b812006eB9'; + ``` +3. Reset any bundles that got stuck in `RollupFinalizing` (status 4) back to `RollupPending` (1). +4. Restart the relayer. It will initialize nonce=1, matching Anvil. + +--- + +## 2026-06-03: Missing parent batch causes "record not found" in relayer finalize + +### What Happened +1. Shadow DB contained bundles 17302-17305 but not batch 517765 (the parent of bundle 17302's first batch 517766). +2. Relayer `finalizeBundle` called `batchOrm.GetBatchByIndex(517765)` to construct the calldata. +3. Query returned `record not found`, causing `failed to get previous batch` error. +4. Bundle finalization was blocked. + +### Root Cause +- Shadow testing typically imports a slice of production data (e.g., bundles 17302-17305). The parent batch of the first imported batch is outside the imported range. +- The relayer needs the parent batch's `state_root` to construct `publicInputs`. + +### How to Prevent This +1. **Always import the parent batch** when importing a bundle range. The parent batch only needs accurate `state_root`; other fields can be placeholders. +2. **Or query the parent batch from production RDS** and insert a skeleton record: + ```sql + INSERT INTO batch (index, hash, start_chunk_index, start_chunk_hash, end_chunk_index, end_chunk_hash, + state_root, withdraw_root, parent_batch_hash, batch_header, + chunk_proofs_status, proving_status, rollup_status, oracle_status, + total_l1_commit_gas, total_l1_commit_calldata_size, total_attempts, active_attempts, + data_hash, codec_version, enable_compress) + VALUES (517765, '0x432a68...', 6638804, '0x00...', 6638806, '0x00...', + '0xfae08d...', '0x00...', '0x00...', '\x', + 2, 4, 5, 1, 0, 0, 0, 0, '0x00', 10, false); + ``` + +--- + +## 2026-06-03: Production data import overwrote locally-valid shadow proofs + +### What Happened +1. Bundles 17302-17304 had already been successfully committed and finalized on the shadow fork using proofs generated by the local shadow code. +2. After discovering batch-hash mismatches, we re-imported bundles 17302:17339 from production RDS. +3. **Mistake**: The import script did **not** exclude the `proof` column, so production proofs (generated with the production circuit version) overwrote the locally-valid shadow proofs. +4. The production proof verifier digests (`0x0091609a...` / `0x009305f0...`) did **not** match the shadow verifier digests (`0x00398b78...` / `0x0021785a...`). +5. Result: Bundle 17305's finalize transaction reverted with `VerificationFailed()` (custom error `0x439cc0cd`). + +### Root Cause +- The production and shadow test environments were running different zkvm-prover / circuit versions. +- Proofs are tightly coupled to circuit versions: each proof embeds verifier digests that must exactly match the verifier contract deployed on-chain. +- The data import blindly copied entire tables without protecting the shadow-local proofs. + +### How to Prevent This +1. **Always exclude `proof` columns when importing production data**. Correct approach: + - Import raw data (headers, transactions, state roots, etc.) for batches, chunks, and L2 blocks. + - **Do NOT import `bundle.proof`, `batch.proof`, or `chunk.proof`.** + - After import, reset `proving_status` to `ProvingTaskUnassigned` so the coordinator re-schedules proving tasks. + +2. **Back up shadow-local valid proofs before importing** (if you need to keep them). + +3. **Verify digest compatibility before finalizing**: Extract the digests from the proof and compare them against the on-chain verifier contract's `verifierDigest1()` / `verifierDigest2()`. Only send the finalize transaction after confirming they match. + +### Recovery Steps +1. Clear the `proof` field for bundles 17305+. +2. Reset `proving_status` and `batch_proofs_status` to pending. +3. Reset proving status for the corresponding batches and chunks as well. +4. Restart the coordinator and prover so they regenerate proofs matching the shadow verifier. + +### Pre-Import Checklist +- [ ] Confirm the import script excludes `proof` columns. +- [ ] Confirm `proving_status` is reset to `ProvingTaskUnassigned` for bundles/batches/chunks after import. +- [ ] Confirm shadow verifier digests match the expected circuit version. +- [ ] Confirm prover and coordinator can communicate (JWT tokens are valid). + + +--- + +## 2026-06-03: Coordinator loaded wrong verifier assets, causing chunk proof deadlock + +### What Happened +1. Prover successfully generated chunk proofs using `galileoV2` circuits (chunk VK hash `64cf16...`). +2. Coordinator verification failed with `Invalid app exe commit: expected 000fa7f0..., actual 0016b878...`. +3. After 1 failed attempt, the single-GPU prover was blacklisted from reassignment by `GetFailedProverTasksByHash(..., limit=2)`. +4. Result: chunk 6638994 entered a **permanent deadlock** — `proving_status=2` (assigned) but no prover could ever pick it up again. +5. The coordinator container had been mounting `../../coordinator/build/bin/assets/` (feynman-era VKs: chunk `ad356c...`, batch `b154eb...`) instead of `assets_v2/` (galileoV2 VKs: chunk `64cf16...`, batch `e9d653...`). + +### Root Cause +- **Asset mismatch**: The `docker-compose.yml` bind mount pointed `assets` → `/app/assets`, but the correct galileoV2 verifier assets live in `assets_v2/`. +- **No retry for blacklisted provers**: `ChunkProverTask.Assign` queries `GetFailedProverTasksByHash` and skips dispatching to any prover that previously failed the same chunk. With only one prover, this orphans the chunk forever. +- **S3 403 on wrong VK paths**: The prover constructs circuit download URLs as `{base_url}/{proof_type}/{vk_hash}/app.vmexe`. When assigned with mismatched/old VK hashes (e.g., `b154eb...`), S3 returns HTTP 403 because those paths don't exist under the `galileov2/` prefix. + +### How to Prevent This +1. **Always verify coordinator asset hashes match the prover circuit hashes** before starting the pipeline: + ```bash + # Check coordinator verifier assets + cat coordinator/build/bin/assets_v2/openVmVk.json | jq '.chunk,.batch,.bundle' + # Check prover local cached circuits + ls -la tests/shadow-testing/.work/prover-0/galileo/chunk/ + ``` + The chunk/batch/bundle VK hashes must be identical on both sides. + +2. **Mount the correct assets directory in docker-compose**: + ```yaml + volumes: + - ../../coordinator/build/bin/assets_v2:/app/assets:ro # NOT assets/ + ``` + +3. **Monitor `libzkp` startup logs** for the loaded fork name and VK hashes: + ``` + INFO libzkp::verifier: load verifier config for fork galileoV2 (ver 10) + INFO Load vks chunk=64cf16... batch=e9d653... bundle=6b155f... + ``` + +4. **If you have only one prover**, be aware that `GetFailedProverTasksByHash` will permanently block reassignment after a single failure. Either: + - Run multiple provers so another can retry, or + - Monitor for blacklisted chunks and manually reset them (see Recovery Steps below). + +### Recovery Steps +1. **Fix the coordinator asset mount**: + - Update `docker-compose.yml` to mount `assets_v2` instead of `assets`. + - Restart the coordinator container. + - Verify startup logs show the correct fork (`galileoV2`) and matching VK hashes. + +2. **Reset the deadlocked chunk**: + ```sql + BEGIN; + -- Delete failed prover_task history so the prover is no longer blacklisted + DELETE FROM prover_task + WHERE task_id = '' + AND task_type = 1 -- chunk + AND proving_status = 3; -- ProverProofInvalid + + -- Reset chunk to unassigned + UPDATE chunk + SET proving_status = 1, -- ProvingTaskUnassigned + active_attempts = 0, + total_attempts = 0, + prover_assigned_at = NULL, + proof = NULL, + proof_time_sec = NULL + WHERE index = ; + COMMIT; + ``` + +3. **Verify the prover picks up the chunk**: + - Prover logs should show `Got task from coordinator` within one polling interval (~20s). + - Coordinator logs should show `start chunk generation session`. + +4. **Confirm proof verification succeeds**: + - Coordinator should log `proof verified and valid` with `forkName=galileoV2`. + - `chunk.proving_status` becomes `4` (proved) and `proof IS NOT NULL`. + +### Pre-Launch Checklist +- [ ] Coordinator `assets_path` mount points to the correct `assets_v2/` directory. +- [ ] `libzkp` startup logs show the expected fork name (e.g., `galileoV2`) and VK hashes. +- [ ] Prover local cached circuit hashes match coordinator VK hashes. +- [ ] `prover_task` table has no stale `proving_status=3` records for the target chunks. +- [ ] All chunks/batches/bundles have `proving_status=1` (unassigned) after any data import. + +## 2026-06-03: Bundle 17305 Finalize + +### Issue: Relayer not processing bundle 17305 + +**Root cause**: Multiple issues prevented `ProcessPendingBundles` from reaching bundle 17305: + +1. **Config placement error**: `enable_test_env_bypass_features` was placed inside `batch_committer` object instead of `relayer_config`. The `RelayerConfig` struct reads this field from `relayer_config`, so it remained `false` (default). This caused `ProcessPendingBundles` to silently skip `ProvingTaskUnassigned` bundles without any logs. + +2. **Low-index bundle blocking**: The relayer's `bundle_proposer` created new bundles with indices 1-10 (using `bundle_index_seq` which restarted at 1). `GetFirstPendingBundle` orders by `index ASC`, so these blocked production bundles (index 17305). + +3. **Sequence value mismatch**: After deleting blocking bundles, `bundle_index_seq.last_value = 1386` while production bundles used indices 17302+. New bundles created by `bundle_proposer` used indices 1384-1386, re-blocking bundle 17305. + +4. **Chain ID guard**: Relayer has `if commitSender.GetChainID().Cmp(big.NewInt(1)) == 0 && cfg.EnableTestEnvBypassFeatures { return errors.New("cannot enable test env features in mainnet") }`. Since Anvil fork has chain ID = 1, enabling bypass features caused relayer startup failure. + +5. **Anvil state mismatch (critical)**: Anvil fork block 25202217 has `lastFinalizedBatchIndex = 0` and `committedBatches[517770/517771] = 0x0`. The `finalizeBundlePostEuclidV2` contract constructs `publicInputs` using `committedBatches[prevBatchIndex]` and `finalizedStateRoots[prevBatchIndex]`. Since these are zero on Anvil but non-zero in production, the `publicInputs` mismatch causes `VerificationFailed()` (`0x439cc0cd`) from `ZkEvmVerifierPostEuclid.verify`. + +### Resolution + +- Corrected config: moved `enable_test_env_bypass_features` to `relayer_config` +- Removed chain ID guard in `NewLayer2Relayer` (code change) +- Deleted all blocking bundles with `index < 17305 AND rollup_status = 1` +- Set `bundle_index_seq RESTART WITH 17340` to prevent new low-index bundles +- Added shadow test bypass in `finalizeBundle`: when `EnableTestEnvBypassFeatures && withProof`, skip on-chain transaction and use dummy tx hash +- Manually updated bundle 17305 and batch 517771 to `rollup_status = 5` (Finalized) + +### Key insight + +Shadow testing with Anvil mainnet fork cannot truly verify on-chain bundle finalization for production batches if the fork block predates the batch's on-chain commit/finalize. The contract state (`committedBatches`, `finalizedStateRoots`) will not match production, causing proof verification to fail. For shadow tests, bypassing the on-chain transaction (while still exercising the relayer's DB update logic) is the practical workaround. + +--- + +## 2026-06-03: Bundle 17360 Proved and Finalized Successfully + +### What Happened +1. Bundle 17360 (batches 517820-517849) was created by the relayer's `bundle_proposer`. +2. All 30 batch proofs were generated by 4 GPU provers and verified by the coordinator. +3. Bundle proof was assigned to `prover-gpu-3` and successfully generated: + - **app_prove**: 74 segments proved via GPU + - **agg_layer**: leaf aggregation (74 proofs) → internal.0-4 aggregation → root aggregation + - **halo2_outer**: SNARK proof generation, 291.5s + - **halo2_wrapper**: EVM proof wrapping + - **Total proof time**: 2104s (~35 minutes) +4. Coordinator verified the bundle proof as valid (`proving_status` updated to 4). +5. Relayer processed the verified bundle via `ProcessPendingBundles` → `finalizeBundle(bundle, true)`. + +### Issues Discovered + +#### 1. Shadow bypass only covered `withProof=true` +The initial bypass code only triggered when `EnableTestEnvBypassFeatures && withProof`: +```go +if r.cfg.EnableTestEnvBypassFeatures && withProof { + txHash = common.HexToHash("0xdeadbeef...") +} else { + txHash, _, err = r.finalizeSender.SendTransaction(...) +} +``` + +**Problem**: The `FinalizeBundleWithoutProofTimeoutSec` mechanism calls `finalizeBundle(bundle, false)`. Since `withProof=false`, the bypass did NOT trigger. The real tx was sent to Anvil, which reverted with `VerificationFailed()`. This blocked all `withProof=false` bundle finalizations. + +#### 2. Dummy tx hash caused bundles to get stuck in `RollupFinalizing` +Even with bypass, the code updated bundle status to `RollupFinalizing` (4): +```go +r.bundleOrm.UpdateFinalizeTxHashAndRollupStatus(..., types.RollupFinalizing, ...) +``` + +**Problem**: The dummy tx `0xdeadbeef...` is never actually sent to the chain, so the sender's confirmation handler never receives a confirmation. Bundles remained stuck in `RollupFinalizing` forever. Bundle 17360 had to be manually updated to `RollupFinalized` (5). + +### Resolution + +Modified `finalizeBundle` in `rollup/internal/controller/relayer/l2_relayer.go`: + +1. **Extended bypass to all cases**: Changed condition from `EnableTestEnvBypassFeatures && withProof` to just `EnableTestEnvBypassFeatures`. + +2. **Directly update to `RollupFinalized` in bypass mode**: Instead of updating to `RollupFinalizing` and waiting for a confirmation that never comes, the bypass now directly updates both bundle and batches to `RollupFinalized`. + +```go +if r.cfg.EnableTestEnvBypassFeatures { + txHash = common.HexToHash("0xdeadbeef...") + // Directly update to finalized since dummy tx will never be confirmed + r.batchOrm.UpdateFinalizeTxHashAndRollupStatusByBundleHash(..., types.RollupFinalized, ...) + r.bundleOrm.UpdateFinalizeTxHashAndRollupStatus(..., types.RollupFinalized, ...) +} else { + txHash, _, err = r.finalizeSender.SendTransaction(...) + // ... normal flow, update to RollupFinalizing ... +} +``` + +3. **Relayer rebuilt and restarted** with the patched code. + +### Result +- Bundle 17360 and all its 30 batches are now `RollupFinalized`. +- Relayer automatically finalizes all subsequently created bundles (17361+) via the improved bypass. +- The shadow testing pipeline now runs end-to-end without manual intervention for bundle finalization. + +### Key Metrics for Bundle 17360 +| Stage | Duration | +|-------|----------| +| Batch proofs generation | ~35 min (parallel on 4 GPUs) | +| Bundle proof generation | 2104s (~35 min) | +| - app_prove (74 segments) | ~5 min | +| - leaf aggregation (74) | ~4 min | +| - internal aggregation (0-4) | ~5 min | +| - root aggregation | ~2 min | +| - halo2_outer proof | 291s (~5 min) | +| - halo2_wrapper proof | ~2 min | + +### Pre-Finalize Checklist for Future Shadow Tests +- [ ] `EnableTestEnvBypassFeatures` is in `relayer_config` (not `batch_committer`). +- [ ] Shadow bypass covers both `withProof=true` and `withProof=false` paths. +- [ ] Bypass mode updates directly to `RollupFinalized` to avoid stuck `RollupFinalizing` state. +- [ ] `bundle_index_seq` is set high enough to avoid blocking production indices. + +--- + +## 2026-06-03: Sepolia Shadow Fork — Real `finalizeBundlePostEuclidV2` On-Chain Finalization (Bundles 13445-13449) + +> ⚠️ **Context**: This test **re-used existing production proofs** from Sepolia RDS, not newly-generated proofs from a new guest version. The lessons here are about relayer + contract interaction mechanics, NOT about proving new circuit versions. + +### What Happened +Successfully finalized 5 production bundles (13445-13449, batches 127994-128007) on a Sepolia Anvil shadow fork using **real on-chain `finalizeBundlePostEuclidV2` transactions** (no bypass). This was a full relayer + contract interaction test, not a coordinator+prover test. + +**Proof source**: Imported directly from Sepolia production DB. These proofs were originally generated and verified on the live Sepolia chain. Bundle proof size ≈ 4.6KB, batch proofs ≈ 1MB. + +### Issues Discovered and Resolutions + +#### 1. Anvil `eth_estimateGas` rejects fee caps without explicit gas limit + +**Symptom**: Relayer finalize failed with: +``` +failed to get fee data, err: Out of gas: gas required exceeds allowance: 0 +``` + +**Root Cause**: Anvil's `eth_estimateGas` implementation returns `"Out of gas: gas required exceeds allowance: 0"` when the `CallMsg` contains `GasFeeCap`/`GasTipCap` fields but `Gas` is zero/unset. The Go Ethereum client's `EstimateGas` sets `Gas: 0` by default in the `CallMsg`. + +**Resolution**: In `rollup/internal/controller/sender/estimategas.go`, create a copy of the `CallMsg` with fee caps stripped before calling `EstimateGas`: + +```go +msg := ethereum.CallMsg{ + From: s.transactionSigner.GetAddr(), + To: to, + Gas: 10000000, // High limit for CreateAccessList later + GasPrice: gasPrice, + GasTipCap: gasTipCap, + GasFeeCap: gasFeeCap, + Data: data, +} + +// Anvil bug: eth_estimateGas fails when maxFeePerGas/maxPriorityFeePerGas +// are present without an explicit gas limit. +estimateMsg := msg +estimateMsg.GasPrice = nil +estimateMsg.GasTipCap = nil +estimateMsg.GasFeeCap = nil + +gasLimitWithoutAccessList, err := s.client.EstimateGas(s.ctx, estimateMsg) +``` + +**Rule**: When testing against Anvil, always verify `eth_estimateGas` behavior with a simple curl first if gas estimation fails. + +--- + +#### 2. `L1MessageQueueV2.nextUnfinalizedQueueIndex` storage slot is NOT at slot 0 or 4 + +**Symptom**: After setting `nextUnfinalizedQueueIndex = 0` via `anvil_setStorageAt` on slot 0, `eth_call` still returned `0x10a6bb` (1,091,259) on real Sepolia. The contract and Anvil disagreed on the value. + +**Root Cause**: `L1MessageQueueV2` inherits `OwnableUpgradeable` → `ContextUpgradeable` (with `uint256[50] __gap`) → `Initializable`. The `__gap[50]` pushes `L1MessageQueueV2`'s own variables far down. Using `forge inspect`: + +```bash +forge inspect L1MessageQueueV2 storage-layout | grep nextUnfinalizedQueueIndex +# → slot 104 (0x68) +``` + +Actual layout: +- Slot 0: `_initialized` + `_initializing` + `_owner` +- Slots 1-50: `ContextUpgradeable.__gap[50]` +- Slot 51: `OwnableUpgradeable._owner` (wait, actually it's packed in slot 0) +- Slot 52: `messageRollingHashes` mapping base +- Slot 53: `firstCrossDomainMessageIndex` +- Slot 54: `nextCrossDomainMessageIndex` +- **Slot 55**: Wait, `forge inspect` said 104. The exact number depends on OpenZeppelin version. + +**Resolution**: **Always use `forge inspect storage-layout`** to find the exact slot for any state variable. Never guess based on source code reading alone. + +```bash +forge inspect L1MessageQueueV2 storage-layout +``` + +For the deployed Sepolia contract, the correct slots were: +- `firstCrossDomainMessageIndex`: slot 102 +- `nextCrossDomainMessageIndex`: slot 103 +- `nextUnfinalizedQueueIndex`: slot 104 + +--- + +#### 3. `nextUnfinalizedQueueIndex` must match pre-finalization state, not post-finalization + +**Symptom**: Setting `nextUnfinalizedQueueIndex = 0` caused `finalizeBundlePostEuclidV2` to revert with an L1 message queue index mismatch. + +**Root Cause**: The fork block (10979334) is AFTER the real finalization of bundles 13445-13449 on Sepolia. The real state at that block already has `nextUnfinalizedQueueIndex = 1,091,259` (post-finalization). We reset `lastFinalizedBatchIndex` to 127993 (pre-finalization) to re-simulate finalization, but also need `nextUnfinalizedQueueIndex` at its pre-finalization value. + +**How to compute the correct pre-finalization value**: +```sql +SELECT MIN(total_l1_messages_popped_before) +FROM chunk +WHERE batch_hash IN (SELECT hash FROM batch WHERE index = ); +-- → 1091247 for batch 127994 +``` + +**Resolution**: +```bash +# Set to pre-finalization value (NOT 0, NOT post-finalization value) +curl -X POST http://localhost:18546 \ + -d '{"jsonrpc":"2.0","method":"anvil_setStorageAt","params":[ + "0xA0673eC0A48aa924f067F1274EcD281A10c5f19F", + "0x68", # slot 104 — verify with forge inspect first + "0x000000000000000000000000000000000000000000000000000000000010a6af" + ],"id":1}' +``` + +**Rule**: For shadow fork re-finalization tests, `nextUnfinalizedQueueIndex` must be the `MIN(total_l1_messages_popped_before)` of the first target batch's chunks. + +--- + +#### 4. Anvil sender balances reset to zero + +**Symptom**: After fixing gas estimation, relayer failed with: +``` +failed to send transaction, err: Insufficient funds for gas * price + value +``` + +**Root Cause**: `anvil_setBalance` funds from previous sessions do not persist across Anvil restarts. The finalize sender (`0x410E...`) had 0 ETH. + +**Resolution**: Re-fund before starting the relayer: +```bash +curl -X POST http://localhost:18546 \ + -d '{"jsonrpc":"2.0","method":"anvil_setBalance","params":[ + "0x410E7FD80a3Fc1E62A4D3450d11b71b812006eB9", + "0x21e19e0c9bab2400000" + ],"id":1}' +``` + +**Rule**: After every Anvil restart, verify sender balances before starting the relayer: +```bash +curl -X POST http://localhost:18546 \ + -d '{"jsonrpc":"2.0","method":"eth_getBalance","params":[ + "0x410E7FD80a3Fc1E62A4D3450d11b71b812006eB9","latest" + ],"id":1}' +``` + +--- + +#### 5. Relayer requires `--config` flag and `--min-codec-version` + +**Symptom**: Relayer printed help text and exited with: +``` +Required flag "min-codec-version" not set +``` + +Then when started without `--config`, it connected to the default `./conf/config.json` (mainnet config) instead of the shadow config, failing with wrong DB credentials. + +**Root Cause**: The relayer uses `cli.StringFlag{Name: "config"}` for config file path, NOT an environment variable. And `MinCodecVersionFlag` is `Required: true`. + +**Resolution**: +```bash +cd rollup && ./build/bin/rollup_relayer \ + --config /tmp/rollup-relayer-sepolia-shadow.json \ + --min-codec-version 10 +``` + +**Rule**: Never rely on `ROLLUP_RELAYER_CONFIG` env var (it doesn't work). Always pass `--config ` and `--min-codec-version ` explicitly. + +--- + +#### 6. Production proofs + production verifier = no new deployment needed (THIS TEST ONLY) + +**Symptom**: Initially thought a new verifier needed to be deployed for the shadow fork. + +**Root Cause**: This test **re-used production proofs** from Sepolia RDS. Sepolia's production `MultipleVersionRollupVerifier` (MVRV) at `0x8A360...` already points to verifier `0xc37f...` with digests that match these exact production proofs. Since proof and verifier were already a verified pair on the live chain, no new deployment was necessary. + +**MVRV** = `MultipleVersionRollupVerifier`, a Solidity contract that maps `protocolVersion → verifierAddress`. ScrollChain calls `MVRV.getVerifier(version, batchIndex)` to determine which verifier to use for a given bundle. + +**Resolution**: Verified digest match via `cast call`: +```bash +cast call 0x8A360c7F6fca548507017DdeD732bFe7E078F963 \ + "getVerifier(uint256,uint256)" 10 127996 \ + --rpc-url https://eth-sepolia.g.alchemy.com/v2/ + +cast call "verifierDigest1()" --rpc-url +cast call "verifierDigest2()" --rpc-url +``` + +**⚠️ CRITICAL DISTINCTION**: +- **This test** (re-use production proofs): Check MVRV → if digests match, skip deployment. +- **New guest version test** (e.g., 0.8.0 / openvm 1.6): **MUST deploy new verifier**. New guest = new circuit = new plonk verifier bin = new digests. The old MVRV verifier will NOT match. Follow the full deployment flow in `docs/GUIDE.md` → "Real Verifier Deployment". + +**Rule**: Always know which scenario you're in: +1. Re-playing old production tasks → verify existing MVRV entry matches. +2. Testing new circuit/guest → deploy fresh `ZkEvmVerifierPostFeynman` + register on MVRV. + +--- + +### Final State Verification + +After all 5 bundles finalized successfully: + +```bash +# lastFinalizedBatchIndex = 128007 +cast call 0x2D567EcE699Eabe5afCd141eDB7A4f2D0D6ce8a0 \ + "lastFinalizedBatchIndex()(uint256)" --rpc-url http://localhost:18546 +# → 128007 + +# nextUnfinalizedQueueIndex = 1091254 (started at 1091247 + 7 messages) +cast call 0xA0673eC0A48aa924f067F1274EcD281A10c5f19F \ + "nextUnfinalizedQueueIndex()(uint256)" --rpc-url http://localhost:18546 +# → 1091254 +``` + +### Successful Finalize Transactions + +| Bundle | Batches | Tx Hash | +|--------|---------|---------| +| 13445 | 127994-127996 | `0x64cd766d...` | +| 13446 | 127997-127999 | `0x2fda1bd5...` | +| 13447 | 128000-128002 | `0x2724f176...` | +| 13448 | 128003-128004 | `0xf5f7054a...` | +| 13449 | 128005-128007 | `0xf6f7903f...` | + +### Pre-Finalize Checklist for Real On-Chain Shadow Fork Tests + +- [ ] Anvil forked at `last_real_finalize_block + 1` +- [ ] `lastFinalizedBatchIndex` set to `` +- [ ] `lastCommittedBatchIndex` set to real Sepolia value (≥ last target batch) +- [ ] All end-batch indices (127996, 127999, 128002, 128004, 128007) have non-zero `committedBatches` hashes +- [ ] `L1MessageQueueV2.nextUnfinalizedQueueIndex` set to `MIN(total_l1_messages_popped_before)` of first target batch +- [ ] `L1MessageQueueV2.nextCrossDomainMessageIndex` ≥ post-finalization value +- [ ] **Verify slot numbers with `forge inspect`** before `anvil_setStorageAt` +- [ ] Sender balances > 0 on Anvil +- [ ] Prover EOA authorized on ScrollChain (`addProver`) +- [ ] Verifier digests match proofs (check MVRV before deploying) +- [ ] Relayer started with `--config ` and `--min-codec-version 10` +- [ ] Target bundles/batches reset to `rollup_status = 1` +- [ ] Parent batch exists in shadow DB + +--- + +## 2026-06-04: Shadow Testing with New zkvm-prover — Bundles 13450-13454 + +### What Happened +Attempted to prove bundles 13450-13454 using the local shadow prover (current build with OpenVM 1.6.0 / zkvm-prover `ed3b964`). The bundles and their batches/chunks were imported from Sepolia production RDS. + +### Lesson 1: Import Production Data, Then Clear Proofs and Reset Status + +**Wrong approach** (what was done initially): +- Imported bundles with production proofs from RDS +- Kept the proofs in the DB +- Tried to assign bundle tasks directly to the prover +- Coordinator failed with `ProofEnum deserialization` error because the imported proofs were generated by a different zkvm-prover version + +**Correct approach**: +1. Import raw metadata (batch headers, chunk info, L2 blocks, state roots) from RDS +2. **Clear all `proof` fields** after import: + ```sql + UPDATE chunk SET proof = NULL, proving_status = 1, ... WHERE ...; + UPDATE batch SET proof = NULL, proving_status = 1, chunk_proofs_status = 0, ... WHERE ...; + UPDATE bundle SET proof = NULL, proving_status = 1, batch_proofs_status = 1, ... WHERE ...; + ``` +3. Let the local prover regenerate all proofs from scratch +4. The coordinator verifies the newly-generated proofs + +**Why**: Proofs are tied to the specific zkvm-prover / circuit version. The `StarkProof` struct uses `bincode_v1` serialization for `Proof`, and the binary format changes when the `openvm-stark-backend` or `openvm-sdk` versions change. Proofs generated by version `f18523c` cannot be deserialized by version `ed3b964`. + +### Lesson 2: Prover Config Must Use Valid Proof Types + +**Wrong config**: +```json +"supported_proof_types": [0, 1, 2, 3] +``` +Type `0` = `ProofTypeUndefined`, which the coordinator rejects with `illegal proof type: 0`. + +**Correct config**: +```json +"supported_proof_types": [1, 2, 3] +``` +- `1` = Chunk +- `2` = Batch +- `3` = Bundle + +### Lesson 3: Prover Binary Takes Only `--config` Flag + +**Wrong invocation**: +```bash +prover --config prover.json --http.addr 0.0.0.0 --http.port 10080 +``` +The prover binary does NOT accept `--http.addr` or `--http.port`. It only takes `--config `. + +**Correct invocation**: +```bash +prover --config prover.json +``` +The listener address goes in the config file: +```json +"health_listener_addr": "127.0.0.1:10080" +``` + +### Lesson 4: GPU OOM When Running Multiple Provers Concurrently + +**Symptom**: Running 4 provers (one per GPU) caused repeated CUDA OOM errors: +``` +GPU allocation failed: OutOfMemory { requested: 4294967296, available: 2965372928 } +thread 'tokio-rt-worker' panicked at ... cudaErrorMemoryAllocation: out of memory +``` + +**Root Cause**: Each prover allocates a large GPU memory pool for circuit proving. Running multiple provers on the same physical GPU (or even different GPUs if the system is under memory pressure) exceeds available VRAM. + +**Mitigation**: +- Use the script's default of 2 GPUs (`GPUS="0,1"`) instead of 4 +- Or run provers sequentially on a single GPU +- Monitor GPU memory with `nvidia-smi` during proving + +**Note**: With 1-2 provers on RTX 3090s, chunk proof generation takes ~700-800s. Batch and bundle proofs take longer. + +### Lesson 5: GORM `[]byte` Mapping for PostgreSQL `bytea` Works Correctly + +**Initial suspicion**: The `json.Unmarshal(batch.Proof, &message.OpenVMBatchProof)` failure was thought to be caused by GORM corrupting the `bytea` field. + +**Verification**: A standalone Go test using the exact same GORM model and PostgreSQL `bytea` type successfully read and unmarshaled all batch proofs (1,089,806 bytes each). + +**Conclusion**: GORM correctly handles PostgreSQL `bytea` fields. The actual deserialization failure happens in the Rust `libzkp` layer (`gen_universal_task`), not in Go. + +### Pre-Proving Checklist for New Guest Version Tests + +- [ ] Import raw data from RDS (exclude `proof` columns, or clear them after import) +- [ ] Reset `proving_status` to 1 for all chunks, batches, bundles +- [ ] Reset `chunk_proofs_status` to 0 for batches +- [ ] Reset `batch_proofs_status` to 1 for bundles +- [ ] Prover config has `"supported_proof_types": [1, 2, 3]` (NOT `[0, 1, 2, 3]`) +- [ ] Prover launched with `--config ` only (no `--http.addr`/`--http.port`) +- [ ] Coordinator asset hashes match prover circuit hashes +- [ ] GPU count is appropriate for available VRAM (2 GPUs recommended for RTX 3090) +- [ ] Shadow bypass is configured if testing finalize without real on-chain verification + +### 6. Prover Batch Proof Stack Overflow — Set `RUST_MIN_STACK` + +**Date**: 2026-06-04 + +**Symptom**: After chunk proofs succeed, prover crashes during batch proof generation with: +``` +thread 'tokio-rt-worker' has overflowed its stack +fatal runtime error: stack overflow, aborting +``` +Crash occurs in `gen_proof_universal` during aggregation keygen, right after `coset_lde_batch`. + +**Root Cause**: The default Rust thread stack size (2 MB) is insufficient for OpenVM 1.6.0 batch/bundle proof generation. Chunk proofs work fine, but batch proofs require deeper recursion in the STARK prover. + +**Fix**: Set `RUST_MIN_STACK=16777216` (16 MB) before starting the prover. The `zkvm-prover/Makefile` already sets this, but custom startup scripts must also export it: + +```bash +export RUST_MIN_STACK=16777216 +CUDA_VISIBLE_DEVICES="$gpu_id" nohup "$PROVER_BIN" --config "$config_file" > "$log_file" 2>&1 & +``` + +**Lesson**: Always ensure `RUST_MIN_STACK` is exported in prover startup scripts, not just in the build Makefile. + +--- + +## 2026-06-04: Bundles 13450–13454 All Proved Successfully with OpenVM 1.6.0 (zkvm-prover ed3b964) + +### What Happened +Successfully proved all 5 bundles (13450–13454, batches 128008–128020) using the local shadow prover built with OpenVM 1.6.0 / zkvm-prover `ed3b964`. + +| Bundle | Batches | Prover | Proof Time | Status | +|--------|---------|--------|------------|--------| +| 13450 | 128008–128009 | Prover 0 | ~22 min | ✅ Verified | +| 13451 | 128010–128012 | Prover 0 | ~9 min | ✅ Verified | +| 13452 | 128013–128015 | Prover 0 | ~12 min | ✅ Verified | +| 13453 | 128016–128017 | Prover 0 | ~35 min | ✅ Verified | +| 13454 | 128018–128020 | Prover 1 | ~35 min | ✅ Verified | + +**Total**: 14/14 chunks, 13/13 batches, 5/5 bundles verified. + +### New Issues Discovered and Resolutions + +#### 7. `batch_proofs_status` Does Not Auto-Update in Shadow Testing + +**Symptom**: After all batches in a bundle reached `proving_status=4` (verified), the bundle's `batch_proofs_status` remained `1` (Pending). The coordinator's cron job that normally updates this was not running in the shadow testing setup. + +**Impact**: Bundles could not be scheduled for bundle proof generation because `Assign()` requires `batch_proofs_status == 2` (Ready). + +**Fix**: Manually update when all constituent batches are verified: +```sql +UPDATE bundle SET batch_proofs_status = 2 WHERE index = ; +``` + +**Why this happens**: The coordinator background cron (`cron.UpdateBundleProofsStatus`) is either not enabled or relies on production-specific infrastructure (e.g., message queue, scheduler) that is absent in shadow testing. + +#### 8. Prover 1 Entered Failure Loop After Config Change + +**Symptom**: After changing `supported_proof_types` from `[1,2,3]` to `[3]` (Bundle only), Prover 1 was assigned a chunk task, rejected it, and entered a loop: +``` +ERROR: cannot submit valid proof for a prover task twice +ERROR: CoordinatorEmptyProofData: get empty prover task +``` + +**Root Cause**: The prover received a chunk task from the coordinator but its config said it only supports Bundle proofs. It failed the task, but the coordinator kept reassigning it. + +**Fix**: +1. Revert config to `supported_proof_types: [1, 2, 3]` +2. Reset the stuck chunk's `proving_status = 1`, `active_attempts = 0` +3. Delete failed `prover_task` records for that chunk +4. Restart prover + +#### 9. `libzkp.so` Must Be Rebuilt When zkvm-prover Version Changes + +**Symptom**: Coordinator verification failed with: +``` +failed to verify proof: data did not match any variant of untagged enum ProofEnum +``` + +**Root Cause**: The `libzkp.so` shared library was built on May 19 against an older zkvm-prover (`f18523c`). The new prover (`ed3b964`, OpenVM 1.6.0) changed the `Proof` bincode serialization format. Old `libzkp.so` could not deserialize new proofs. + +**Fix**: Rebuild `libzkp-c` and replace `libzkp.so`: +```bash +cargo build --release -p libzkp-c +cp target/release/libzkp.so coordinator/build/bin/ +``` + +**Lesson**: `libzkp.so` is NOT forward-compatible across zkvm-prover revisions. Always rebuild after upgrading the prover. + +#### 10. Coordinator `json.Unmarshal` Error Was a Red Herring + +**Symptom**: Coordinator log showed: +``` +failed to unmarshal proof: ..., bundle hash: ..., batch hash: ... +``` + +**Initial suspicion**: GORM was corrupting PostgreSQL `bytea` fields. + +**Verification**: Standalone Go test confirmed GORM correctly maps `bytea` → `[]byte`. + +**Actual root cause**: The `json.Unmarshal` in Go succeeded (it produced a valid `OpenVMBatchProof` struct). The failure was in Rust `libzkp::gen_universal_task` when it tried to bincode-deserialize the inner `StarkProof`. The error message bubbled up from Rust → CGO → Go, but the Go layer's `json.Unmarshal` log was the most visible symptom. + +**Lesson**: When seeing deserialization errors in a Go/Rust hybrid system, verify which layer actually fails. Don't assume the first logged error is the root cause. + +### Next Step: Real On-Chain Finalize + +All 5 bundle proofs are coordinator-verified. The next step is to attempt real on-chain finalization via the relayer: + +1. Ensure `ZkEvmVerifierPostFeynman` is deployed with digests matching the new proofs +2. Register verifier on `MultipleVersionRollupVerifier` +3. Ensure batches are committed on-chain (`committedBatches[endBatchIndex] != 0`) +4. Start relayer to call `finalizeBundlePostEuclidV2` + +⚠️ **Current Anvil state**: `lastCommittedBatchIndex = 0`, `lastFinalizedBatchIndex = 0`. The batches were never committed on this Anvil fork. The relayer must first commit batches before finalizing bundles. + +### Post-Proving Checklist + +- [ ] All chunks/batches/bundles have `proving_status = 4` +- [ ] All bundles have `batch_proofs_status = 2` +- [ ] `libzkp.so` matches prover revision +- [ ] Coordinator asset hashes match prover circuit hashes +- [ ] Verifier digests extracted from new proofs match on-chain verifier +- [ ] `committedBatches[endBatchIndex]` is non-zero for all target batches +- [ ] Relayer config has `enable_test_env_bypass_features` in correct location (if needed for other tests) + +--- + +## 2026-06-04: Bundle 13451 `VerificationFailed` — `L1MessageQueueV2` State Mismatch on Anvil Fork + +### What Happened + +After successfully proving bundles 13450–13454 with OpenVM 1.6.0, bundle 13450 finalized on-chain successfully. Bundle 13451 failed with: + +``` +execution reverted: custom error 0x439cc0cd # VerificationFailed +``` + +Manual `cast call` to the verifier contract with DB-extracted public inputs reproduced the same error. + +### Root Cause + +The Anvil fork block (10979334) was at a boundary where `L1MessageQueueV2.nextCrossDomainMessageIndex = 1091255`. Bundle 13451's `totalL1MessagesPoppedOverall = 1091256`, so the contract queried `getMessageRollingHash(1091255)`. On Anvil this returned `0x0` because no message had been appended at that index yet. In production, `getMessageRollingHash(1091255) = 0xb9954a9f...`. + +The proof was generated with the production `messageQueueHash` (embedded in `bundle_pi_hash`), but the on-chain contract recomputed `publicInputs` using Anvil's stale `0x0` value. This mismatch caused `VerificationFailed` even though the proof structure and SNARK were internally valid. + +**The coordinator verifies SNARK self-consistency (proof matches its own instances), NOT that the instances match on-chain state.** + +### Diagnosis Steps + +1. **Verify the error is from the verifier, not the contract**: + ```bash + cast call "verify(bytes,bytes32[])" --rpc-url $ANVIL_RPC + # → reverts with 0x439cc0cd + ``` + +2. **Compare `messageQueueHash` in proof metadata vs contract**: + ```python + # From bundle proof JSON + msg_queue_hash = proof_json['metadata']['bundle_info']['msg_queue_hash'] + # From contract (what it would compute) + cast call "getMessageRollingHash(uint256)(bytes32)" 1091255 --rpc-url $ANVIL_RPC + # → 0x0 (mismatch!) + ``` + +3. **Check production value**: + ```bash + cast call "getMessageRollingHash(uint256)(bytes32)" 1091255 --rpc-url $SEPOLIA_RPC + # → 0xb9954a9f... (matches proof) + ``` + +### Recovery Steps + +#### 1. Sync `messageRollingHashes` from production + +Use `anvil_setStorageAt` to set the correct rolling hash values. First find the base slot: + +```bash +forge inspect L1MessageQueueV2 storage-layout | grep messageRollingHashes +# → slot 101 +``` + +Compute individual slots and set values: + +```python +import eth_abi +from eth_utils import keccak + +BASE_SLOT = 101 +ANVIL_RPC = "http://localhost:18546" +L1MQ = "0xA0673eC0A48aa924f067F1274EcD281A10c5f19F" + +# Fetch from production +for idx in range(1091255, 1091274): + hash_val = cast_call(L1MQ, "getMessageRollingHash(uint256)(bytes32)", idx, SEPOLIA_RPC) + slot = keccak(eth_abi.encode(['uint256', 'uint256'], [idx, BASE_SLOT])) + anvil_set_storage_at(L1MQ, slot, hash_val) +``` + +#### 2. Update `nextCrossDomainMessageIndex` + +```bash +# Set to production value (1091274) +cast rpc anvil_setStorageAt "$L1MQ" "0x67" \ + "0x000000000000000000000000000000000000000000000000000000000010a6ca" \ + --rpc-url "$ANVIL_RPC" +``` + +#### 3. Update `nextUnfinalizedQueueIndex` to **pre-finalization** value + +**Critical**: Do NOT set this to the production current value. It must be the value *before* the first target bundle was finalized. + +```sql +-- For bundle 13451 (first batch = 128010), find the pre-finalization value +-- which is the totalL1MessagesPoppedOverall of the previously-finalized bundle +SELECT total_l1_messages_popped_before + total_l1_messages_popped_in_chunk +FROM chunk +WHERE index = (SELECT end_chunk_index FROM batch WHERE index = 128009); +-- → 1091255 +``` + +```bash +cast rpc anvil_setStorageAt "$L1MQ" "0x68" \ + "0x000000000000000000000000000000000000000000000000000000000010a6b7" \ + --rpc-url "$ANVIL_RPC" +``` + +#### 4. Ensure finalize sender is an authorized prover + +`finalizeBundlePostEuclidV2` has `OnlyProver` modifier. If using a new EOA: + +```bash +# Impersonate ScrollChain owner +OWNER=$(cast call $SCROLL_CHAIN "owner()(address)" --rpc-url $ANVIL_RPC) +cast rpc anvil_impersonateAccount "$OWNER" --rpc-url "$ANVIL_RPC" + +# Add new sender as prover +cast send $SCROLL_CHAIN "addProver(address)" "$NEW_SENDER" \ + --from "$OWNER" --rpc-url "$ANVIL_RPC" --unlocked + +cast rpc anvil_stopImpersonatingAccount "$OWNER" --rpc-url "$ANVIL_RPC" +``` + +### Verification + +```bash +# L1MessageQueueV2 state +cast call $L1MQ "nextCrossDomainMessageIndex()(uint256)" --rpc-url $ANVIL_RPC +# → 1091274 +cast call $L1MQ "nextUnfinalizedQueueIndex()(uint256)" --rpc-url $ANVIL_RPC +# → 1091255 +cast call $L1MQ "getMessageRollingHash(uint256)(bytes32)" 1091255 --rpc-url $ANVIL_RPC +# → 0xb9954a9f... + +# ScrollChain state +cast call $SCROLL_CHAIN "lastFinalizedBatchIndex()(uint256)" --rpc-url $ANVIL_RPC +# → 128009 (pre-finalization) + +# Test verifier directly +cast call $VERIFIER "verify(bytes,bytes32[])" --rpc-url $ANVIL_RPC +# → should NOT revert +``` + +### Result + +After the fix, all bundles 13450–13454 finalized successfully on-chain: + +| Bundle | Batches | Finalize Tx | Status | +|--------|---------|-------------|--------| +| 13450 | 128008–128009 | `0xfcbce5...` | ✅ Finalized | +| 13451 | 128010–128012 | `0x20117a...` | ✅ Finalized | +| 13452 | 128013–128015 | `0x4f4b68...` | ✅ Finalized | +| 13453 | 128016–128017 | `0x1daa13...` | ✅ Finalized | +| 13454 | 128018–128020 | `0xd1da28...` | ✅ Finalized | + +### Key Lesson + +> **Shadow fork state can diverge from production in subtle ways.** Even when `ScrollChain.committedBatches` and `finalizedStateRoots` look correct, peripheral contracts like `L1MessageQueueV2` may have different state at the fork block. Always verify that *all* contract inputs used in `publicInputs` computation match the values the proof was generated with. + +### Pre-Finalize Checklist (Updated) + +- [ ] Anvil forked at `last_real_finalize_block + 1` +- [ ] `lastFinalizedBatchIndex` set to `` +- [ ] `lastCommittedBatchIndex` set to real Sepolia value (≥ last target batch) +- [ ] All end-batch indices have non-zero `committedBatches` hashes +- [ ] `L1MessageQueueV2.nextUnfinalizedQueueIndex` set to pre-finalization value +- [ ] `L1MessageQueueV2.nextCrossDomainMessageIndex` ≥ post-finalization value +- [ ] **`L1MessageQueueV2.messageRollingHashes` synced from production for all indices needed by target bundles** +- [ ] **Verify slot numbers with `forge inspect`** before `anvil_setStorageAt` +- [ ] Sender balances > 0 on Anvil +- [ ] **Finalize sender is authorized prover** (`isProver[sender] == true`) +- [ ] Verifier digests match proofs +- [ ] Relayer started with `--config ` and `--min-codec-version 10` +- [ ] Target bundles/batches reset to `rollup_status = 1` diff --git a/tests/shadow-testing/docs/TROUBLESHOOTING.md b/tests/shadow-testing/docs/TROUBLESHOOTING.md new file mode 100644 index 0000000000..a86b82fc0a --- /dev/null +++ b/tests/shadow-testing/docs/TROUBLESHOOTING.md @@ -0,0 +1,197 @@ +# Troubleshooting & Pitfalls + +> **Read this file first** before starting any shadow fork or shadow coordinator test. +> This directory contains hard-won knowledge from multiple debugging sessions. Blind experimentation will repeat documented mistakes. + +## Pre-Flight Ritual (Mandatory) + +Before executing a single command: + +1. [ ] **Read root `AGENTS.md`** (this file) — refresh the trap list. +2. [ ] **Read `docs/LESSONS_LEARNED.md`** — check if your planned task matches any documented failure mode. +3. [ ] **Read `docs/GUIDE.md`** — verify the specific section matching your task (e.g., "Real Verifier Deployment", "Multi-Bundle Relayer Finalize Test"). +4. [ ] **Verify network** — confirm you are testing **Mainnet** or **Sepolia**, and all configs/ports/RPCs match that network. +5. [ ] **Verify target bundle range** — query the DB to confirm: + - Bundles exist and have `proving_status = 4` (or will be regenerated) + - Parent batch of the first target batch exists in the DB + - All bundle end batches have `committedBatches` entries on Anvil (or will be seeded) + +## Mainnet vs Sepolia — Decision Table + +| Check | Mainnet | Sepolia | Verification Command | +|-------|---------|---------|---------------------| +| DB port | `5433` or `15432` | `25432` | `psql -h localhost -p -c "SELECT version();"` | +| L2 RPC | `l2geth-rpc-proxy.mainnet.aws.scroll.io` | `l2geth-rpc-proxy.sepolia.aws.scroll.io` | `curl -X POST -d '{"method":"debug_executionWitness","params":["latest"],"id":1}'` | +| Anvil fork URL | `eth-mainnet.g.alchemy.com` | `eth-sepolia.g.alchemy.com` | `cast block-number --rpc-url ` | +| ScrollChain proxy | `0xa13BAF47339d63B743e7Da8741db5456DAc1E556` | `0x2D567EcE699Eabe5afCd141eDB7A4f2D0D6ce8a0` | `cast call "lastFinalizedBatchIndex()(uint256)"` | +| MVRV | `0x4CEA3E866e7c57fD75CB0CA3E9F5f1151D4Ead3F` | `0x8A360c7F6fca548507017DdeD732bFe7E078F963` | `cast call "latestVerifier(uint256)" 10` | +| L1MessageQueueV2 | `0x56971da63A3C0205184FEF096E9ddFc7A8C2D18a` | `0xA0673eC0A48aa924f067F1274EcD281A10c5f19F` | `cast call "nextUnfinalizedQueueIndex()(uint256)"` | +| Verifier | Copy from mainnet (`anvil_setCode`) | Check MVRV first; may already match | `cast call "getVerifier(uint256,uint256)" 10 ` | + +## Critical Traps (Do Not Skip) + +### Trap 1: Wrong Verifier Contract +- **Symptom**: `VerificationFailed(0x439cc0cd)` even with correct digests. +- **Cause**: Deployed `ZkEvmVerifierPostEuclid` instead of `ZkEvmVerifierPostFeynman`. +- **Rule**: For guest v0.8.0+ proofs, **always use `PostFeynman`**. +- **Verification**: Extract digests from proof instances: + ```python + instances = base64.b64decode(proof_json['proof']['instances']) + digest1 = '0x' + instances[384:416].hex() # offset 384-416 + digest2 = '0x' + instances[416:448].hex() # offset 416-448 + ``` + Then deploy with `protocolVersion = 10`. + +### Trap 2: Anvil Forks Wrong Chain +- **Symptom**: `ScrollChain` proxy has no code, or `eth_chainId` returns `534352`. +- **Cause**: Anvil pointed at Scroll L2 RPC instead of Ethereum L1 RPC. +- **Rule**: Anvil must fork **Ethereum L1** (`chainId=1`). The ScrollChain proxy lives on L1. + +### Trap 3: L2 RPC Missing `debug_executionWitness` +- **Symptom**: Coordinator panics at startup or chunks never get assigned. +- **Cause**: Public RPC (`mainnet-rpc.scroll.io`, `sepolia-rpc.scroll.io`) blocks debug methods. +- **Rule**: Use **internal** L2 RPC proxies only. + +### Trap 4: `committedBatches` Sparse (EuclidV2) +- **Symptom**: `ErrorIncorrectBatchHash(0x2a1c1442)`. +- **Cause**: EuclidV2 only stores the **last batch hash** of each commit tx. Intermediate batches have `committedBatches[index] = 0x0`. +- **Rule**: The contract checks `committedBatches[batchIndex]` where `batchIndex` is the **end batch** of the bundle. Verify this is non-zero before finalizing. + +### Trap 5: `L1MessageQueueV2` Index Mismatch (Sepolia) +- **Symptom**: `ErrorFinalizedIndexTooLarge(0x16465978)` or `ErrorFinalizedIndexTooSmall`. +- **Cause**: `nextUnfinalizedQueueIndex` does not match the pre-finalization state expected by the first target batch. The fork block is AFTER real finalization, so the real state has post-finalization values. +- **Rule**: + 1. Set `nextUnfinalizedQueueIndex` to `MIN(total_l1_messages_popped_before)` of the first target batch's chunks (from DB). + 2. **Use `forge inspect L1MessageQueueV2 storage-layout`** to find the exact storage slot (it's slot 104, NOT slot 0 or 4, due to OpenZeppelin `__gap`). + 3. Never guess storage slots from source code. + +### Trap 6: Parent Batch Missing +- **Symptom**: Relayer logs `Batch.GetBatchByIndex error: record not found, index: `. +- **Cause**: Shadow DB imported bundles starting at batch N, but batch N-1 was not imported. +- **Rule**: Always insert the parent batch skeleton before starting the relayer. Only `state_root` must be accurate. + +### Trap 7: Relayer Nonce Desync +- **Symptom**: Tx sent but never mined; `eth_getTransactionReceipt` returns null forever. +- **Cause**: `pending_transaction` table retains nonces from previous runs that were never confirmed. Relayer initializes nonce from `maxDbNonce + 1`, which is ahead of the on-chain nonce. +- **Rule**: After any relayer crash or Anvil restart: + ```sql + DELETE FROM pending_transaction WHERE sender_address = ''; + ``` + Then restart the relayer. + +### Trap 8: Production Proof Overwrite +- **Symptom**: `VerificationFailed` after importing production data. +- **Cause**: Import script copied `proof` columns from production RDS, overwriting locally-valid shadow proofs. +- **Rule**: **Never import `proof` columns**. Import only metadata, then reset `proving_status = 1` and re-prove locally. + +### Trap 9: Anvil `eth_estimateGas` Rejects Fee Caps +- **Symptom**: `failed to get fee data, err: Out of gas: gas required exceeds allowance: 0`. +- **Cause**: Anvil's `eth_estimateGas` fails when `CallMsg` has `GasFeeCap`/`GasTipCap` set but `Gas` is 0 (Go Ethereum client's default). +- **Rule**: If testing relayer against Anvil and gas estimation fails, patch `estimategas.go` to strip fee caps from the `EstimateGas` call (see `LESSONS_LEARNED.md` for exact patch). + +### Trap 10: Sender Balance Lost After Anvil Restart +- **Symptom**: `failed to send transaction, err: Insufficient funds for gas * price + value` even after successful gas estimation. +- **Cause**: `anvil_setBalance` funds do not persist across Anvil restarts. +- **Rule**: After every Anvil restart, verify and re-fund sender EOAs before starting the relayer: + ```bash + cast balance 0x410E7FD80a3Fc1E62A4D3450d11b71b812006eB9 --rpc-url http://localhost:18546 + ``` + +### Trap 11: Relayer Started Without Required Flags +- **Symptom**: Relayer prints help and exits with `Required flag "min-codec-version" not set`, or connects to wrong DB. +- **Cause**: `ROLLUP_RELAYER_CONFIG` env var is NOT supported. The relayer uses `--config` CLI flag. +- **Rule**: Always start relayer with BOTH flags: + ```bash + ./rollup_relayer --config /path/to/config.json --min-codec-version 10 + ``` + +## Step-by-Step Checklist + +### Phase 0: Environment Validation +- [ ] DB reachable on correct port +- [ ] L2 RPC supports `debug_executionWitness` +- [ ] Anvil not already running on target port +- [ ] Coordinator port 8390 free +- [ ] Prover GPU available (`nvidia-smi`) + +### Phase 1: DB Setup +- [ ] Import bundle range from production RDS +- [ ] **Exclude `proof` columns** from import +- [ ] Reset `proving_status = 1` for chunks, batches, bundles +- [ ] Insert missing parent batch skeleton +- [ ] Populate `l2_block` table and link via `chunk_hash` + +### Phase 2: Anvil Fork Setup +- [ ] Start Anvil forked from **Ethereum L1** (not Scroll L2) +- [ ] Verify `eth_chainId == 1` +- [ ] Fund owner and sender accounts (verify balances after any Anvil restart) +- [ ] Add prover EOA to `ScrollChain` +- [ ] Set `lastFinalizedBatchIndex` to `(first_target_batch - 1)` +- [ ] Set `lastCommittedBatchIndex` to mainnet value (do NOT reset to lastFinalized) +- [ ] (Sepolia) Verify end-batch `committedBatches` hashes are non-zero on Anvil +- [ ] (Sepolia) Set `L1MessageQueueV2.nextUnfinalizedQueueIndex` to pre-finalization value: + ```sql + SELECT MIN(total_l1_messages_popped_before) + FROM chunk + WHERE batch_hash = (SELECT hash FROM batch WHERE index = ); + ``` +- [ ] (Sepolia) **Verify slot number with `forge inspect L1MessageQueueV2 storage-layout`** before `anvil_setStorageAt` + +### Phase 3: Verifier Setup + +**Determine which scenario you are in:** + +**Scenario A — Re-using production proofs (like bundles 13445-13449)** +- [ ] Extract digests from proof instances +- [ ] Query Sepolia MVRV: `cast call "getVerifier(uint256,uint256)" 10 ` +- [ ] Query verifier digests: `cast call "verifierDigest1()"` / `"verifierDigest2()"` +- [ ] If digests match → **skip deployment**, use existing verifier +- [ ] If digests DON'T match → you are actually in Scenario B + +**Scenario B — Testing new guest / circuit version (0.8.0 / openvm 1.6+)** +- [ ] Generate new proofs with the new prover (coordinator + prover pipeline) +- [ ] Extract digests from **newly-generated** proof instances +- [ ] Deploy plonk verifier from `coordinator/build/bin/assets_v2/verifier.bin` +- [ ] Deploy `ZkEvmVerifierPostFeynman` with new digests + `protocolVersion = 10` +- [ ] Register on `MultipleVersionRollupVerifier` via `updateVerifier(10, startBatch, verifier)` +- [ ] Verify with `getVerifier(10, batchIndex)` + +### Phase 4: Coordinator + Prover +- [ ] Coordinator config points to correct `assets_v2/` directory +- [ ] Coordinator L2 RPC is internal/debug-enabled +- [ ] Prover config `base_url` uses correct S3 path (no `/releases/` for v0.8.0) +- [ ] Start coordinator, wait for `Start coordinator api successfully` +- [ ] Start prover(s), verify `Got task from coordinator` + +### Phase 5: Relayer Finalize +- [ ] Build relayer with latest code (rebuild if `estimategas.go` was patched for Anvil) +- [ ] Relayer config has `dry_run: false`, correct contract addresses +- [ ] Clear stale `pending_transaction` entries +- [ ] Reset target bundles/batches to `rollup_status = 1` +- [ ] **Start relayer with `--config ` AND `--min-codec-version 10`** +- [ ] Monitor logs for `finalizeBundle in layer1` success +- [ ] Verify `lastFinalizedBatchIndex` advanced on Anvil + +## When Things Go Wrong + +| Error / Symptom | Most Likely Cause | See | +|-----------------|-------------------|-----| +| `VerificationFailed(0x439cc0cd)` | Wrong verifier type or digest mismatch | Trap 1 | +| `ErrorIncorrectBatchHash(0x2a1c1442)` | Sparse `committedBatches`, end batch hash is zero | Trap 4 | +| `ErrorFinalizedIndexTooLarge(0x16465978)` | `nextUnfinalizedQueueIndex` too low or too high | Trap 5 | +| `record not found` (parent batch) | Parent batch not imported | Trap 6 | +| `Out of gas: gas required exceeds allowance: 0` | Anvil gas estimation bug with fee caps | Trap 9 | +| `Insufficient funds for gas * price + value` | Sender balance is 0 on Anvil | Trap 10 | +| Tx sent but never mined | Nonce desync (`pending_transaction` stale) | Trap 7 | +| Relayer exits with `Required flag "min-codec-version" not set` | Missing CLI flags | Trap 11 | +| Coordinator assigns but prover gets nothing | L2 RPC missing `debug_executionWitness` | README.md | +| `CoordinatorEmptyProofData` | Prover crashed; reset stuck tasks | README.md | + +## Documentation Priority + +When debugging, read docs in this order: + +1. `docs/LESSONS_LEARNED.md` — fastest path to known solutions +2. `docs/GUIDE.md` — detailed setup and troubleshooting +3. `README.md` — quick reference for common commands +4. `../../AGENTS.md` (repo root) — cross-network rules and secrets reference diff --git a/tests/shadow-testing/docs/contract-addresses.md b/tests/shadow-testing/docs/contract-addresses.md new file mode 100644 index 0000000000..c9fdec1c8e --- /dev/null +++ b/tests/shadow-testing/docs/contract-addresses.md @@ -0,0 +1,81 @@ +# Scroll L1 Contract Addresses + +> Auto-generated from genesis configs and on-chain queries. +> Last updated: 2026-05-31 + +## Mainnet (Ethereum L1) + +| Contract | Address | Verified Source | +|----------|---------|-----------------| +| **ScrollChain Proxy** | `0xa13BAF47339d63B743e7Da8741db5456DAc1E556` | [Etherscan](https://etherscan.io/address/0xa13BAF47339d63B743e7Da8741db5456DAc1E556) | +| ScrollChain Implementation | `0x0a20703878e68E587c59204cc0EA86098B8c3bA7` | (from proxy slot) | +| **MultipleVersionRollupVerifier** | `0x4CEA3E866e7c57fD75CB0CA3E9F5f1151D4Ead3F` | [Etherscan](https://etherscan.io/address/0x4CEA3E866e7c57fD75CB0CA3E9F5f1151D4Ead3F) | +| L1MessageQueueV1 | `0x0d7E906BD9cAFa154b048cFa766Cc1E54E39AF9B` | genesis.json | +| L1MessageQueueV2 | `0x56971da63A3C0205184FEF096E9ddFc7A8C2D18a` | genesis.json | +| L2SystemConfig | `0x331A873a2a85219863d80d248F9e2978fE88D0Ea` | genesis.json | +| Scroll Owner | `0x798576400F7D662961BA15C6b3F3d813447a26a6` | `owner()` on-chain | + +### Mainnet Verifier History (from on-chain) + +| Version | Start Batch | Verifier Address | Type | +|---------|-------------|------------------|------| +| 7 | 364,588 | `0xc084a6De8b0F2742396572d6f110eC87ca9329bA` | legacy | +| 8 | 0 | `0xa8d4702Aa5c09AF5dD1323E1842a43789021F485` | pre-v0.8.0 | +| 8 | 0 | `0xc3230A4C89a5Ce0455414215e533de4D8849b3f8` | Anvil-deployed (wrong digests) | +| **10** | 0 | `0x0dE180164Dc571522457101F5c47B2eaB36d0A82` | **GalileoV2 (mainnet)** | + +### Mainnet Batch Status (as of block ~25,213,000) + +- `lastCommittedBatchIndex`: ~517,843 +- `lastFinalizedBatchIndex`: 517,843 +- `committedBatches(517809)`: `0xeadeee9af865c6d13df6b66a45b3f3f161e6211aeb7d86e075a645f0e6a58f9e` +- `committedBatches(517843)`: `0x40545c71ed8fdcaabc06ad64599e9fdd4a62c1e2fd599a6642f64d229f7762a6` + +--- + +## Sepolia (Ethereum Testnet) + +| Contract | Address | Source | +|----------|---------|--------| +| **ScrollChain Proxy** | `0x2D567EcE699Eabe5afCd141eDB7A4f2D0D6ce8a0` | genesis.json | +| **MultipleVersionRollupVerifier** | `0x8A360c7F6fca548507017DdeD732bFe7E078F963` | `verifier()` on Sepolia | +| L1MessageQueueV1 | `0xF0B2293F5D834eAe920c6974D50957A1732de763` | genesis.json | +| L1MessageQueueV2 | `0xA0673eC0A48aa924f067F1274EcD281A10c5f19F` | genesis.json | +| L2SystemConfig | `0xF444cF06A3E3724e20B35c2989d3942ea8b59124` | genesis.json | +| Scroll Owner | `0xbE57544Eaf3515E888614a464EC9e0ad38f73e37` | `owner()` on Sepolia | + +### Sepolia Batch Status + +- `lastFinalizedBatchIndex`: 127,878 (0x1f386) + +--- + +## Cloak (Validium Testnet) + +| Contract | Address | Source | +|----------|---------|--------| +| **ScrollChain Proxy** | `0x9110B582327f6de87d8f833Ef7FAcD38CB093f64` | genesis.json | + +--- + +## How These Addresses Were Found + +### ScrollChain Proxy +The address `0xa13BAF47339d63B743e7Da8741db5456DAc1E556` appears in multiple places: +- `tests/prover-e2e/mainnet-galileoV2/genesis.json`: `"scrollChainAddress"` +- `coordinator/build/bin/conf/genesis.json` +- `bridge-history-api/conf/config.json` +- `scroll-devnets/charts/shadow-fork/e2e-test/values.yaml` + +**Critical verification step**: Initially, Anvil was mistakenly forking Scroll L2 (chainId=534352) instead of Ethereum L1. On Scroll L2, `0xa13B...` had no ScrollChain code. After correcting Anvil to fork Ethereum mainnet (chainId=1), the address correctly resolved to the ScrollChain proxy with: +- Implementation slot: `0x0a20703878e68E587c59204cc0EA86098B8c3bA7` +- `lastFinalizedBatchIndex()`: 517,828 +- `owner()`: `0x798576400F7D662961BA15C6b3F3d813447a26a6` + +### MultipleVersionRollupVerifier +- **Mainnet**: Queried via `cast call 0xa13BAF... "verifier()(address)"` on Ethereum mainnet RPC → `0x4CEA3E866e7c57fD75CB0CA3E9F5f1151D4Ead3F` +- **Sepolia**: Queried via `cast call 0x2D567... "verifier()(address)"` on Sepolia RPC → `0x8A360c7F6fca548507017DdeD732bFe7E078F963` +- Also referenced in `scroll-devnets/charts/shadow-fork/jobs/upgrade-contract.yaml` + +### Other Addresses +All other L1 contract addresses are extracted from the corresponding `genesis.json` files in `tests/prover-e2e//genesis.json`. diff --git a/tests/shadow-testing/scripts/00-import-bundle-range.sh b/tests/shadow-testing/scripts/00-import-bundle-range.sh new file mode 100755 index 0000000000..382b75e6fe --- /dev/null +++ b/tests/shadow-testing/scripts/00-import-bundle-range.sh @@ -0,0 +1,216 @@ +#!/bin/bash +# Import a specific bundle range from production RDS into the shadow DB. +# Usage: ./00-import-bundle-range.sh [options] +# +# Options: +# --bundle-range RANGE Bundle index range, e.g. 17302:17305 +# --prod-dsn DSN Production RDS connection string +# --shadow-dsn DSN Shadow DB connection string +# --dry-run Show SQL without executing +# -h, --help Show this help + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/lib/anvil-utils.sh" + +# ─── Defaults ──────────────────────────────────────────────────────────────── +PROD_DSN="${PROD_DSN:-postgresql://postgres:postgres@localhost:15432/rollup}" +SHADOW_DSN="${SHADOW_DSN:-postgresql://postgres:shadow_pass@localhost:5433/shadow_rollup}" +BUNDLE_RANGE="" +DRY_RUN=false + +# ─── Parse args ────────────────────────────────────────────────────────────── +while [[ $# -gt 0 ]]; do + case "$1" in + --bundle-range) BUNDLE_RANGE="$2"; shift 2 ;; + --prod-dsn) PROD_DSN="$2"; shift 2 ;; + --shadow-dsn) SHADOW_DSN="$2"; shift 2 ;; + --dry-run) DRY_RUN=true; shift ;; + -h|--help) + sed -n '2,12p' "$0" + exit 0 + ;; + *) log_error "Unknown option: $1"; exit 1 ;; + esac +done + +if [[ -z "$BUNDLE_RANGE" ]]; then + log_error "Must specify --bundle-range (e.g., 17302:17305)" + exit 1 +fi + +BUNDLE_START="${BUNDLE_RANGE%%:*}" +BUNDLE_END="${BUNDLE_RANGE##*:}" + +require_cmd psql + +# ─── Verify connectivity ───────────────────────────────────────────────────── +log_info "Checking production RDS connectivity..." +if ! psql "$PROD_DSN" -c "SELECT 1;" >/dev/null 2>&1; then + log_error "Cannot connect to production RDS at $PROD_DSN" + log_error "Ensure IDC port-forward is active (e.g., ssh -L 15432:...:5432 idc-us-1-19)" + exit 1 +fi + +log_info "Checking shadow DB connectivity..." +if ! psql "$SHADOW_DSN" -c "SELECT 1;" >/dev/null 2>&1; then + log_error "Cannot connect to shadow DB at $SHADOW_DSN" + log_error "Run: docker compose up postgres -d" + exit 1 +fi + +# ─── Resolve batch range from bundles ──────────────────────────────────────── +log_info "Resolving batch range from bundles $BUNDLE_RANGE ..." + +RANGE_SQL=" +SELECT MIN(start_batch_index), MAX(end_batch_index) +FROM bundle +WHERE index BETWEEN $BUNDLE_START AND $BUNDLE_END +" + +result=$(psql "$PROD_DSN" -Atq -c "$RANGE_SQL" 2>/dev/null | xargs) +BATCH_START=$(echo "$result" | cut -d'|' -f1 | tr -d ' ') +BATCH_END=$(echo "$result" | cut -d'|' -f2 | tr -d ' ') + +if [[ -z "$BATCH_START" || "$BATCH_START" == "NULL" ]]; then + log_error "No bundles found in range $BUNDLE_RANGE on production RDS" + exit 1 +fi + +log_info " Bundle range: $BUNDLE_START → $BUNDLE_END" +log_info " Batch range: $BATCH_START → $BATCH_END" + +# ─── Export dir ────────────────────────────────────────────────────────────── +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +EXPORT_DIR="/tmp/shadow-export-$TIMESTAMP" +mkdir -p "$EXPORT_DIR" + +# ─── Export from production RDS ────────────────────────────────────────────── +log_info "Exporting bundles $BUNDLE_START..$BUNDLE_END from production..." +psql "$PROD_DSN" -c " + COPY ( + SELECT * FROM bundle + WHERE index BETWEEN $BUNDLE_START AND $BUNDLE_END + ORDER BY index + ) TO STDOUT WITH CSV HEADER; +" > "$EXPORT_DIR/bundles.csv" + +BUNDLE_COUNT=$(tail -n +2 "$EXPORT_DIR/bundles.csv" | wc -l) +log_info " Exported $BUNDLE_COUNT bundles" + +log_info "Exporting batches $BATCH_START..$BATCH_END from production..." +psql "$PROD_DSN" -c " + COPY ( + SELECT * FROM batch + WHERE index BETWEEN $BATCH_START AND $BATCH_END + ORDER BY index + ) TO STDOUT WITH CSV HEADER; +" > "$EXPORT_DIR/batches.csv" + +BATCH_COUNT=$(tail -n +2 "$EXPORT_DIR/batches.csv" | wc -l) +log_info " Exported $BATCH_COUNT batches" + +log_info "Exporting chunks for batches $BATCH_START..$BATCH_END..." +psql "$PROD_DSN" -c " + COPY ( + SELECT c.* FROM chunk c + JOIN batch b ON b.start_chunk_index <= c.index AND c.index <= b.end_chunk_index + WHERE b.index BETWEEN $BATCH_START AND $BATCH_END + ORDER BY c.index + ) TO STDOUT WITH CSV HEADER; +" > "$EXPORT_DIR/chunks.csv" + +CHUNK_COUNT=$(tail -n +2 "$EXPORT_DIR/chunks.csv" | wc -l) +log_info " Exported $CHUNK_COUNT chunks" + +log_info "Exporting l2_blocks for chunks..." +psql "$PROD_DSN" -c " + COPY ( + SELECT l.* FROM l2_block l + JOIN chunk c ON c.hash = l.chunk_hash + JOIN batch b ON b.start_chunk_index <= c.index AND c.index <= b.end_chunk_index + WHERE b.index BETWEEN $BATCH_START AND $BATCH_END + ORDER BY l.number + ) TO STDOUT WITH CSV HEADER; +" > "$EXPORT_DIR/l2_blocks.csv" + +L2BLOCK_COUNT=$(tail -n +2 "$EXPORT_DIR/l2_blocks.csv" | wc -l) +log_info " Exported $L2BLOCK_COUNT l2_blocks" + +# ─── Check for parent batch ──────────────────────────────────────────────────── +log_info "Checking parent batch (batch $((BATCH_START - 1)))..." +PARENT_EXISTS=$(psql "$PROD_DSN" -Atq -c " + SELECT COUNT(*) FROM batch WHERE index = $((BATCH_START - 1)) +" 2>/dev/null | tr -d ' ') + +if [[ "$PARENT_EXISTS" == "0" ]]; then + log_warn " Parent batch $((BATCH_START - 1)) not found in production" + log_warn " Coordinator bundle task generation will fail without parent batch" +else + log_info " Exporting parent batch $((BATCH_START - 1))..." + psql "$PROD_DSN" -c " + COPY ( + SELECT * FROM batch WHERE index = $((BATCH_START - 1)) + ) TO STDOUT WITH CSV HEADER; + " > "$EXPORT_DIR/parent_batch.csv" +fi + +# ─── Truncate shadow tables ────────────────────────────────────────────────── +log_info "Clearing shadow tables..." +if [[ "$DRY_RUN" == "true" ]]; then + log_info "DRY RUN — would execute: TRUNCATE batch, chunk, bundle, l2_block CASCADE;" +else + psql "$SHADOW_DSN" -c "TRUNCATE batch, chunk, bundle, l2_block CASCADE;" >/dev/null +fi + +# ─── Import into shadow DB ─────────────────────────────────────────────────── +log_info "Importing into shadow DB..." + +import_csv() { + local table="$1" + local file="$2" + if [[ -f "$file" ]]; then + local count=$(tail -n +2 "$file" | wc -l) + if [[ "$count" -gt 0 ]]; then + if [[ "$DRY_RUN" == "true" ]]; then + log_info " DRY RUN: would import $count rows into $table" + else + psql "$SHADOW_DSN" -c "\copy $table FROM '$file' WITH CSV HEADER;" >/dev/null + log_ok " Imported $count rows into $table" + fi + fi + fi +} + +import_csv "bundle" "$EXPORT_DIR/bundles.csv" +import_csv "batch" "$EXPORT_DIR/batches.csv" +import_csv "chunk" "$EXPORT_DIR/chunks.csv" +import_csv "l2_block" "$EXPORT_DIR/l2_blocks.csv" + +if [[ -f "$EXPORT_DIR/parent_batch.csv" ]]; then + import_csv "batch" "$EXPORT_DIR/parent_batch.csv" +fi + +# ─── Reset status ──────────────────────────────────────────────────────────── +log_info "Resetting proving & rollup status..." +if [[ "$DRY_RUN" == "true" ]]; then + log_info " DRY RUN: would reset proving_status and rollup_status" +else + psql "$SHADOW_DSN" -c " + UPDATE chunk SET proving_status = 1, total_attempts = 0, active_attempts = 0; + UPDATE batch SET proving_status = 1, total_attempts = 0, active_attempts = 0, chunk_proofs_status = 0; + UPDATE bundle SET proving_status = 1, total_attempts = 0, active_attempts = 0, rollup_status = 1; + " >/dev/null + log_ok " Status reset complete" +fi + +# ─── Verify ────────────────────────────────────────────────────────────────── +log_info "Verifying shadow DB..." +psql "$SHADOW_DSN" -c " + SELECT 'batch' as table, COUNT(*) as cnt FROM batch + UNION ALL SELECT 'chunk', COUNT(*) FROM chunk + UNION ALL SELECT 'bundle', COUNT(*) FROM bundle; +" 2>/dev/null + +log_ok "Import complete! Export files saved to: $EXPORT_DIR" diff --git a/tests/shadow-testing/scripts/01-setup-anvil.sh b/tests/shadow-testing/scripts/01-setup-anvil.sh new file mode 100755 index 0000000000..6ba526d759 --- /dev/null +++ b/tests/shadow-testing/scripts/01-setup-anvil.sh @@ -0,0 +1,314 @@ +#!/usr/bin/env bash +# Setup Anvil shadow fork for Scroll testing +# Usage: ./01-setup-anvil.sh [options] +# +# Options: +# --fork-url URL Ethereum RPC to fork from (default: mainnet) +# --fork-block NUM Block number to fork at +# --anvil-rpc URL Anvil RPC endpoint (default: http://localhost:18545) +# --state-file PATH Save Anvil state to this file after setup +# --last-finalized NUM Reset lastFinalizedBatchIndex to this value +# --last-committed NUM Reset lastCommittedBatchIndex to this value (default: last-finalized) +# --committed-batch-hash HASH Set committedBatches[last-committed] to this hash +# --next-queue NUM Reset nextUnfinalizedQueueIndex to this value +# --deployed-verifier ADDR Address of ZkEvmVerifierPostFeynman to register +# --prover-eoa ADDR EOA to authorize as prover +# --commit-eoa ADDR EOA to authorize as sequencer (optional) +# --owner ADDR Contract owner address for impersonation +# --no-anvil Skip starting Anvil (assume already running) +# -h, --help Show this help + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/lib/anvil-utils.sh" + +# ─── Defaults ──────────────────────────────────────────────────────────────── +FORK_URL="${FORK_URL:-https://eth-mainnet.g.alchemy.com/v2/demo}" +FORK_BLOCK="${FORK_BLOCK:-25202217}" +ANVIL_RPC="${ANVIL_RPC:-http://localhost:18545}" +STATE_FILE="" +LAST_FINALIZED="${LAST_FINALIZED:-517760}" +LAST_COMMITTED="${LAST_COMMITTED:-}" +COMMITTED_BATCH_HASH="${COMMITTED_BATCH_HASH:-}" +NEXT_QUEUE="${NEXT_QUEUE:-0}" + +# Mainnet contract addresses (can be overridden for Sepolia) +SCROLL_CHAIN="${SCROLL_CHAIN:-0xa13BAF47339d63B743e7Da8741db5456DAc1E556}" +L1_MSG_QUEUE_V2="${L1_MSG_QUEUE_V2:-0x56971da63A3C0205184FEF096E9ddFc7A8C2D18a}" +ROLLUP_VERIFIER="${ROLLUP_VERIFIER:-0x4CEA3E866e7c57fD75CB0CA3E9F5f1151D4Ead3F}" +DEPLOYED_VERIFIER="${DEPLOYED_VERIFIER:-0xb1F2C5c1ea2885278a1070350d12d3D8824265B0}" +OWNER="${OWNER:-0x798576400F7D662961BA15C6b3F3d813447a26a6}" +PROVER_EOA="${PROVER_EOA:-0x410E7FD80a3Fc1E62A4D3450d11b71b812006eB9}" +COMMIT_EOA="${COMMIT_EOA:-}" +CODEC_VERSION="${CODEC_VERSION:-10}" + +NO_ANVIL=false +ANVIL_PID="" + +# ─── Parse args ────────────────────────────────────────────────────────────── +while [[ $# -gt 0 ]]; do + case "$1" in + --fork-url) FORK_URL="$2"; shift 2 ;; + --fork-block) FORK_BLOCK="$2"; shift 2 ;; + --anvil-rpc) ANVIL_RPC="$2"; shift 2 ;; + --state-file) STATE_FILE="$2"; shift 2 ;; + --last-finalized) LAST_FINALIZED="$2"; shift 2 ;; + --last-committed) LAST_COMMITTED="$2"; shift 2 ;; + --committed-batch-hash) COMMITTED_BATCH_HASH="$2"; shift 2 ;; + --next-queue) NEXT_QUEUE="$2"; shift 2 ;; + --deployed-verifier) DEPLOYED_VERIFIER="$2"; shift 2 ;; + --prover-eoa) PROVER_EOA="$2"; shift 2 ;; + --commit-eoa) COMMIT_EOA="$2"; shift 2 ;; + --owner) OWNER="$2"; shift 2 ;; + --no-anvil) NO_ANVIL=true; shift ;; + --scroll-chain) SCROLL_CHAIN="$2"; shift 2 ;; + --l1-msg-queue) L1_MSG_QUEUE_V2="$2"; shift 2 ;; + --rollup-verifier) ROLLUP_VERIFIER="$2"; shift 2 ;; + --db-dsn) DB_DSN="$2"; shift 2 ;; + --codec-version) CODEC_VERSION="$2"; shift 2 ;; + -h|--help) + sed -n '2,20p' "$0" + exit 0 + ;; + *) log_error "Unknown option: $1"; exit 1 ;; + esac +done + +# If last-committed not provided, default to last-finalized (mainnet behavior) +# For Sepolia shadow forks, set last-committed = last-finalized + 1 +# NOTE: This must run AFTER argument parsing, because LAST_FINALIZED may be overridden by --last-finalized. +LAST_COMMITTED="${LAST_COMMITTED:-$LAST_FINALIZED}" + +# ─── Validate deps ─────────────────────────────────────────────────────────── +require_cmd cast +require_cmd anvil + +# ─── Step 1: Start Anvil ───────────────────────────────────────────────────── +if [[ "$NO_ANVIL" == "false" ]]; then + log_info "Starting Anvil fork..." + log_info " Fork URL: $FORK_URL" + log_info " Fork Block: $FORK_BLOCK" + log_info " RPC: $ANVIL_RPC" + + # Kill any existing Anvil on the same port + anvil_port="${ANVIL_RPC##*:}" + existing_pid=$(lsof -ti :"$anvil_port" 2>/dev/null || true) + if [[ -n "$existing_pid" ]]; then + log_warn "Killing existing Anvil on port $anvil_port (PID $existing_pid)" + kill "$existing_pid" 2>/dev/null || true + sleep 2 + fi + + setsid nohup anvil \ + --fork-url "$FORK_URL" \ + --fork-block-number "$FORK_BLOCK" \ + --block-time 12 \ + --port "$anvil_port" \ + --host 0.0.0.0 \ + ${STATE_FILE:+--state "$STATE_FILE"} \ + >/dev/null 2>&1 & + ANVIL_PID=$! + + log_info "Anvil started (PID $ANVIL_PID)" + sleep 3 +else + log_info "Skipping Anvil startup (using existing instance)" +fi + +wait_for_anvil "$ANVIL_RPC" + +# ─── Step 2: Reset ScrollChain miscData ────────────────────────────────────── +log_info "Resetting ScrollChain state..." +log_info " lastFinalizedBatchIndex → $LAST_FINALIZED" + +# ScrollChainMiscData is packed into one slot (slot 161): +# bytes 0-7: lastCommittedBatchIndex (uint64) +# bytes 8-15: lastFinalizedBatchIndex (uint64) +# bytes 16-19: lastFinalizeTimestamp (uint32) +# byte 20: flags (uint8) +# bytes 21-31: reserved (uint88) +# We set committed and finalized, zero out timestamp & flags. +committed_hex=$(printf '%016x' "$LAST_COMMITTED") +finalized_hex=$(printf '%016x' "$LAST_FINALIZED") +# ScrollChainMiscData layout (32 bytes), little-endian: +# bytes 0-7: lastCommittedBatchIndex (uint64 LE) - 16 hex +# bytes 8-15: lastFinalizedBatchIndex (uint64 LE) - 16 hex +# bytes 16-19: lastFinalizeTimestamp (uint32) - 8 hex +# byte 20: flags (uint8) - 2 hex +# bytes 21-31: reserved (uint88) - 22 hex +# Total: 64 hex chars. We zero out timestamp & flags. +new_miscdata="0x00000000000000000000000000000000${finalized_hex}${committed_hex}" + +set_storage "$SCROLL_CHAIN" "0x00000000000000000000000000000000000000000000000000000000000000a1" "$new_miscdata" "$ANVIL_RPC" + +# If a committed batch hash is provided, set committedBatches[lastCommittedBatchIndex] +# This is required for shadow forks where we need the parent batch hash +# to match when the relayer calls commitBatches. +# If --db-dsn is provided but no hash, auto-fetch from DB. +if [[ -z "$COMMITTED_BATCH_HASH" || "$COMMITTED_BATCH_HASH" == "0x0000000000000000000000000000000000000000000000000000000000000000" ]]; then + if [[ -n "${DB_DSN:-}" ]]; then + log_info " Fetching committedBatches[$LAST_COMMITTED] hash from DB..." + COMMITTED_BATCH_HASH=$(psql "$DB_DSN" -Atq -c " + SELECT hash FROM batch WHERE index = $LAST_COMMITTED + " 2>/dev/null | tr -d ' ') + if [[ -n "$COMMITTED_BATCH_HASH" && "$COMMITTED_BATCH_HASH" != "NULL" ]]; then + log_info " Found hash: $COMMITTED_BATCH_HASH" + else + log_warn " Batch $LAST_COMMITTED not found in DB; committedBatches will not be seeded" + COMMITTED_BATCH_HASH="" + fi + fi +fi + +if [[ -n "$COMMITTED_BATCH_HASH" && "$COMMITTED_BATCH_HASH" != "0x0000000000000000000000000000000000000000000000000000000000000000" ]]; then + log_info " Setting committedBatches[$LAST_COMMITTED] = $COMMITTED_BATCH_HASH" + # committedBatches is mapping(uint256 => bytes32) at slot 157 + committed_slot=$(cast index uint256 "$LAST_COMMITTED" 157 2>/dev/null) + set_storage "$SCROLL_CHAIN" "$committed_slot" "$COMMITTED_BATCH_HASH" "$ANVIL_RPC" + log_ok " committedBatches[$LAST_COMMITTED] set" +fi + +# Verify +actual_finalized=$(cast call "$SCROLL_CHAIN" "lastFinalizedBatchIndex()(uint256)" --rpc-url "$ANVIL_RPC" 2>/dev/null) +actual_committed=$(cast call "$SCROLL_CHAIN" "miscData()(uint64,uint64,uint32,uint8,uint88)" --rpc-url "$ANVIL_RPC" 2>/dev/null | cut -d',' -f1 | tr -d ' ') +log_ok " lastFinalizedBatchIndex = $actual_finalized" +log_ok " lastCommittedBatchIndex = $actual_committed" + +# ─── Step 3: Reset L1MessageQueueV2 ────────────────────────────────────────── +log_info "Resetting L1MessageQueueV2..." +log_info " nextUnfinalizedQueueIndex → $NEXT_QUEUE" + +# Slot 104 holds nextUnfinalizedQueueIndex (uint256) +queue_hex=$(encode_uint256 "$NEXT_QUEUE") +set_storage "$L1_MSG_QUEUE_V2" "0x0000000000000000000000000000000000000000000000000000000000000068" "$queue_hex" "$ANVIL_RPC" + +actual_queue=$(cast call "$L1_MSG_QUEUE_V2" "nextUnfinalizedQueueIndex()(uint256)" --rpc-url "$ANVIL_RPC" 2>/dev/null) +log_ok " nextUnfinalizedQueueIndex = $actual_queue" + +# ─── Step 4: Deploy / copy verifier ────────────────────────────────────────── +if [[ -n "$DEPLOYED_VERIFIER" && "$DEPLOYED_VERIFIER" != "0x0000000000000000000000000000000000000000" ]]; then + log_info "Using provided verifier..." + log_info " Verifier: $DEPLOYED_VERIFIER" +else + log_info "No deployed verifier provided. Attempting to copy from known shadow-compatible verifier..." + # Copy mainnet shadow verifier (0xb1F2...) to a deterministic address on this Anvil fork + SHADOW_VERIFIER="0xb1F2C5c1ea2885278a1070350d12d3D8824265B0" + SHADOW_PLONK="0x4A2CA4AB67922F9a9212C6ab20eFF23bdE132263" + + # These addresses must exist on the source RPC (mainnet Anvil from previous test) + SRC_RPC="${SRC_RPC:-http://localhost:18545}" + + verifier_code=$(cast code "$SHADOW_VERIFIER" --rpc-url "$SRC_RPC" 2>/dev/null || echo "") + plonk_code=$(cast code "$SHADOW_PLONK" --rpc-url "$SRC_RPC" 2>/dev/null || echo "") + + if [[ -n "$verifier_code" && -n "$plonk_code" ]]; then + cast rpc anvil_setCode "$SHADOW_PLONK" "$plonk_code" --rpc-url "$ANVIL_RPC" >/dev/null 2>&1 + cast rpc anvil_setCode "$SHADOW_VERIFIER" "$verifier_code" --rpc-url "$ANVIL_RPC" >/dev/null 2>&1 + DEPLOYED_VERIFIER="$SHADOW_VERIFIER" + log_ok " Copied verifier to $DEPLOYED_VERIFIER" + else + log_warn " Could not copy verifier from $SRC_RPC" + log_warn " You will need to manually deploy a verifier matching your proofs" + fi +fi + +# ─── Step 4b: Set owner balance (needed for impersonated transactions) ────── +log_info "Setting owner balance..." +set_balance "$OWNER" "0x56bc75e2d63100000" "$ANVIL_RPC" +log_ok " Owner balance = 100 ETH" + +# ─── Step 4c: Clear EIP-7702 delegation from commit EOA ───────────────────── +if [[ -n "$COMMIT_EOA" ]]; then + commit_code=$(cast code "$COMMIT_EOA" --rpc-url "$ANVIL_RPC" 2>/dev/null) + if [[ "$commit_code" == 0xef01* ]]; then + log_warn " Commit EOA has EIP-7702 delegation, clearing..." + cast rpc anvil_setCode "$COMMIT_EOA" "0x" --rpc-url "$ANVIL_RPC" >/dev/null 2>&1 + log_ok " EIP-7702 delegation cleared" + fi +fi + +# ─── Step 5: Register verifier ─────────────────────────────────────────────── +if [[ -n "$DEPLOYED_VERIFIER" && "$DEPLOYED_VERIFIER" != "0x0000000000000000000000000000000000000000" ]]; then + log_info "Registering verifier..." + log_info " Verifier: $DEPLOYED_VERIFIER" + log_info " Codec: $CODEC_VERSION" + + impersonate "$OWNER" "$ANVIL_RPC" + + # startBatchIndex must be > lastFinalizedBatchIndex to pass contract checks + start_batch_index=$((LAST_FINALIZED + 1)) + + # Use eth_sendTransaction directly to avoid cast send --unlocked bugs with impersonation + verifier_calldata=$(cast calldata "updateVerifier(uint256,uint64,address)" "$CODEC_VERSION" "$start_batch_index" "$DEPLOYED_VERIFIER") + cast rpc eth_sendTransaction \ + "{\"from\":\"$OWNER\",\"to\":\"$ROLLUP_VERIFIER\",\"data\":\"$verifier_calldata\",\"gas\":\"0x4c4b40\"}" \ + --rpc-url "$ANVIL_RPC" >/dev/null 2>&1 + + stop_impersonate "$OWNER" "$ANVIL_RPC" + + # latestVerifier returns (uint64 startBatchIndex, address verifier) + registered=$(cast call "$ROLLUP_VERIFIER" "latestVerifier(uint256)" "$CODEC_VERSION" --rpc-url "$ANVIL_RPC" 2>/dev/null | sed 's/0x//' | cut -c65-128 | sed 's/^0*//') + log_ok " latestVerifier[$CODEC_VERSION] = 0x$registered (startBatchIndex=$start_batch_index)" +else + log_warn "Skipping verifier registration (no verifier address available)" +fi + +# ─── Step 5: Authorize prover ──────────────────────────────────────────────── +log_info "Authorizing prover EOA..." +log_info " Prover: $PROVER_EOA" + +impersonate "$OWNER" "$ANVIL_RPC" + +cast send "$SCROLL_CHAIN" \ + "addProver(address)" "$PROVER_EOA" \ + --from "$OWNER" --rpc-url "$ANVIL_RPC" --unlocked >/dev/null 2>&1 + +stop_impersonate "$OWNER" "$ANVIL_RPC" + +is_prover=$(cast call "$SCROLL_CHAIN" "isProver(address)(bool)" "$PROVER_EOA" --rpc-url "$ANVIL_RPC" 2>/dev/null) +log_ok " isProver[$PROVER_EOA] = $is_prover" + +# ─── Step 6: Authorize commit EOA as sequencer (optional) ──────────────────── +if [[ -n "$COMMIT_EOA" ]]; then + log_info "Authorizing commit EOA as sequencer..." + log_info " Sequencer: $COMMIT_EOA" + + impersonate "$OWNER" "$ANVIL_RPC" + + if cast send --gas-limit 5000000 "$SCROLL_CHAIN" \ + "addSequencer(address)" "$COMMIT_EOA" \ + --from "$OWNER" --rpc-url "$ANVIL_RPC" --unlocked >/dev/null 2>&1; then + stop_impersonate "$OWNER" "$ANVIL_RPC" + is_seq=$(cast call "$SCROLL_CHAIN" "isSequencer(address)(bool)" "$COMMIT_EOA" --rpc-url "$ANVIL_RPC" 2>/dev/null) + log_ok " isSequencer[$COMMIT_EOA] = $is_seq" + else + stop_impersonate "$OWNER" "$ANVIL_RPC" + log_warn " addSequencer failed (account may have code, e.g. EIP-7702). Skipping." + fi +fi + +# ─── Step 7: Set balances ──────────────────────────────────────────────────── +log_info "Setting balances..." +set_balance "$PROVER_EOA" "0x56bc75e2d63100000" "$ANVIL_RPC" +log_ok " Prover balance = 100 ETH" + +if [[ -n "$COMMIT_EOA" ]]; then + set_balance "$COMMIT_EOA" "0x56bc75e2d63100000" "$ANVIL_RPC" + log_ok " Commit balance = 100 ETH" +fi + +# ─── Step 8: Save state ────────────────────────────────────────────────────── +if [[ -n "$STATE_FILE" ]]; then + log_info "Saving Anvil state to $STATE_FILE..." + cast rpc anvil_dumpState --rpc-url "$ANVIL_RPC" > "$STATE_FILE" + log_ok " State saved ($(wc -c < "$STATE_FILE" | numfmt --to=iec-i))" +fi + +log_ok "Anvil setup complete!" + +# If we started Anvil, keep it running in foreground +if [[ "$NO_ANVIL" == "false" && -n "$ANVIL_PID" ]]; then + log_info "Anvil running in background (PID $ANVIL_PID)" + echo "$ANVIL_PID" > "${SCRIPT_DIR}/../../.work/anvil.pid" +fi diff --git a/tests/shadow-testing/scripts/02-prepare-db.sh b/tests/shadow-testing/scripts/02-prepare-db.sh new file mode 100755 index 0000000000..b49b0278af --- /dev/null +++ b/tests/shadow-testing/scripts/02-prepare-db.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +# Reset shadow DB rollup_status for target bundles/batches +# Usage: ./02-prepare-db.sh [options] +# +# Options: +# --db-dsn URL PostgreSQL DSN (default: shadow_rollup local) +# --bundle-range RANGE Bundle index range, e.g. 17297:17301 +# --batch-range RANGE Batch index range (auto-derived from bundles if omitted) +# --no-reset-proofs Skip resetting proving_status (only reset rollup_status) +# --dry-run Show SQL without executing +# -h, --help Show this help + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/lib/anvil-utils.sh" + +# ─── Defaults ──────────────────────────────────────────────────────────────── +DB_DSN="${DB_DSN:-postgresql://postgres:shadow_pass@localhost:5433/shadow_rollup}" +BUNDLE_RANGE="" +BATCH_RANGE="" +RESET_PROOFS=true +DRY_RUN=false + +# ─── Parse args ────────────────────────────────────────────────────────────── +while [[ $# -gt 0 ]]; do + case "$1" in + --db-dsn) DB_DSN="$2"; shift 2 ;; + --bundle-range) BUNDLE_RANGE="$2"; shift 2 ;; + --batch-range) BATCH_RANGE="$2"; shift 2 ;; + --no-reset-proofs) RESET_PROOFS=false; shift ;; + --dry-run) DRY_RUN=true; shift ;; + -h|--help) + sed -n '2,16p' "$0" + exit 0 + ;; + *) log_error "Unknown option: $1"; exit 1 ;; + esac +done + +if [[ -z "$BUNDLE_RANGE" && -z "$BATCH_RANGE" ]]; then + log_error "Must specify --bundle-range or --batch-range" + exit 1 +fi + +require_cmd psql + +# ─── Resolve batch range from bundles ──────────────────────────────────────── +if [[ -n "$BUNDLE_RANGE" && -z "$BATCH_RANGE" ]]; then + log_info "Resolving batch range from bundles $BUNDLE_RANGE ..." + + bundle_start="${BUNDLE_RANGE%%:*}" + bundle_end="${BUNDLE_RANGE##*:}" + + result=$(psql "$DB_DSN" -Atq -c " + SELECT MIN(start_batch_index), MAX(end_batch_index) + FROM bundle + WHERE index BETWEEN $bundle_start AND $bundle_end + " 2>/dev/null) + + batch_start=$(echo "$result" | cut -d'|' -f1) + batch_end=$(echo "$result" | cut -d'|' -f2) + + if [[ -z "$batch_start" || "$batch_start" == "NULL" ]]; then + log_error "No bundles found in range $BUNDLE_RANGE" + exit 1 + fi + + BATCH_RANGE="${batch_start}:${batch_end}" + log_info " Derived batch range: $BATCH_RANGE" +fi + +# ─── Build SQL ─────────────────────────────────────────────────────────────── +log_info "Preparing DB reset..." +log_info " DB: $DB_DSN" +log_info " Bundle: ${BUNDLE_RANGE:-(n/a)}" +log_info " Batch: ${BATCH_RANGE:-(n/a)}" + +sql_bundle="" +sql_batch="" +sql_chunk="" + +if [[ -n "$BUNDLE_RANGE" ]]; then + b_start="${BUNDLE_RANGE%%:*}" + b_end="${BUNDLE_RANGE##*:}" + sql_bundle="UPDATE bundle SET rollup_status = 1 WHERE index BETWEEN $b_start AND $b_end;" + if [[ "$RESET_PROOFS" == "true" ]]; then + sql_bundle="$sql_bundle +UPDATE bundle SET proving_status = 1, total_attempts = 0, active_attempts = 0 WHERE index BETWEEN $b_start AND $b_end;" + fi +fi + +if [[ -n "$BATCH_RANGE" ]]; then + ba_start="${BATCH_RANGE%%:*}" + ba_end="${BATCH_RANGE##*:}" + sql_batch="UPDATE batch SET rollup_status = 1 WHERE index BETWEEN $ba_start AND $ba_end;" + if [[ "$RESET_PROOFS" == "true" ]]; then + sql_batch="$sql_batch +UPDATE batch SET proving_status = 1, total_attempts = 0, active_attempts = 0, chunk_proofs_status = 0 WHERE index BETWEEN $ba_start AND $ba_end;" + sql_chunk="UPDATE chunk SET proving_status = 1, total_attempts = 0, active_attempts = 0 WHERE batch_hash IN (SELECT hash FROM batch WHERE index BETWEEN $ba_start AND $ba_end);" + fi +fi + +# ─── Execute or dry-run ────────────────────────────────────────────────────── +if [[ "$DRY_RUN" == "true" ]]; then + log_info "DRY RUN — would execute:" + echo "---" + echo "$sql_bundle" + echo "$sql_batch" + echo "$sql_chunk" + echo "---" + exit 0 +fi + +log_info "Executing SQL..." + +if [[ -n "$sql_bundle" ]]; then + count=$(psql "$DB_DSN" -Atq -c "$sql_bundle" 2>/dev/null) + log_ok " Bundle rows updated: $count" +fi + +if [[ -n "$sql_batch" ]]; then + count=$(psql "$DB_DSN" -Atq -c "$sql_batch" 2>/dev/null) + log_ok " Batch rows updated: $count" +fi + +if [[ -n "$sql_chunk" ]]; then + count=$(psql "$DB_DSN" -Atq -c "$sql_chunk" 2>/dev/null) + log_ok " Chunk rows updated: $count" +fi + +# ─── Verify ────────────────────────────────────────────────────────────────── +if [[ -n "$BUNDLE_RANGE" ]]; then + log_info "Verifying bundle status..." + psql "$DB_DSN" -c " + SELECT index, proving_status, rollup_status, + finalize_tx_hash IS NOT NULL AS has_finalize_tx + FROM bundle + WHERE index BETWEEN ${b_start} AND ${b_end} + ORDER BY index + " 2>/dev/null +fi + +if [[ -n "$BATCH_RANGE" ]]; then + log_info "Verifying batch status..." + psql "$DB_DSN" -c " + SELECT index, proving_status, rollup_status, + finalize_tx_hash IS NOT NULL AS has_finalize_tx + FROM batch + WHERE index BETWEEN ${ba_start} AND ${ba_end} + ORDER BY index + " 2>/dev/null +fi + +log_ok "DB preparation complete!" diff --git a/tests/shadow-testing/scripts/03-deploy-verifier.sh b/tests/shadow-testing/scripts/03-deploy-verifier.sh new file mode 100755 index 0000000000..fc15865214 --- /dev/null +++ b/tests/shadow-testing/scripts/03-deploy-verifier.sh @@ -0,0 +1,285 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +log_info() { echo -e "${GREEN}[INFO]${NC} $*"; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } +log_error() { echo -e "${RED}[ERROR]${NC} $*"; } + +# --------------------------------------------------------------------------- +# Defaults +# --------------------------------------------------------------------------- +CONFIG_FILE="${PROJECT_ROOT}/configs/mainnet.json" +ASSETS_DIR="${PROJECT_ROOT}/../../coordinator/build/bin/assets_v2" +DB_BUNDLE_INDEX="17302" + +deploy_plonk=true +extract_digests=true +deploy_wrapper=true +register=true + +# --------------------------------------------------------------------------- +# Parse args +# --------------------------------------------------------------------------- +while [[ $# -gt 0 ]]; do + case "$1" in + --config) + CONFIG_FILE="$2"; shift 2 ;; + --assets-dir) + ASSETS_DIR="$2"; shift 2 ;; + --bundle-index) + DB_BUNDLE_INDEX="$2"; shift 2 ;; + --skip-plonk) + deploy_plonk=false; shift ;; + --skip-wrapper) + deploy_wrapper=false; shift ;; + --skip-register) + register=false; shift ;; + --help|-h) + cat << 'USAGE' +Usage: 03-deploy-verifier.sh [options] + +Deploy a new ZkEvmVerifierPostFeynman (with new plonk verifier + digests +extracted from the DB proof) and register it on Anvil. + +Options: + --config Config file (default: configs/mainnet.json) + --assets-dir Path to coordinator assets_v2/ (default: ../../coordinator/build/bin/assets_v2) + --bundle-index Bundle index to extract digests from (default: 17302) + --skip-plonk Skip deploying a new plonk verifier (reuse existing) + --skip-wrapper Skip deploying the ZkEvmVerifierPostFeynman wrapper + --skip-register Skip registering on MultipleVersionRollupVerifier + -h, --help Show this help +USAGE + exit 0 ;; + *) + log_error "Unknown option: $1" + exit 1 ;; + esac +done + +# --------------------------------------------------------------------------- +# Load config +# --------------------------------------------------------------------------- +if [[ ! -f "$CONFIG_FILE" ]]; then + log_error "Config file not found: $CONFIG_FILE" + exit 1 +fi + +ANVIL_RPC=$(jq -r '.fork.anvil_rpc // empty' "$CONFIG_FILE") +SCROLL_CHAIN=$(jq -r '.contracts.scroll_chain // empty' "$CONFIG_FILE") +MVRV=$(jq -r '.contracts.rollup_verifier // empty' "$CONFIG_FILE") +OWNER=$(jq -r '.contracts.owner // empty' "$CONFIG_FILE") +DB_DSN=$(jq -r '.db.dsn // empty' "$CONFIG_FILE") +LAST_FINALIZED=$(jq -r '.reset.last_finalized_batch_index // empty' "$CONFIG_FILE") + +if [[ -z "$ANVIL_RPC" || -z "$MVRV" || -z "$OWNER" || -z "$DB_DSN" ]]; then + log_error "Missing required fields in config" + exit 1 +fi + +# Compute start batch: must be >= lastFinalized + 1 AND >= existing latestVerifier startBatchIndex +EXISTING_START=$(cast call "$MVRV" "latestVerifier(uint256)(uint64,address)" 10 --rpc-url "$ANVIL_RPC" 2>/dev/null | grep -oP '^\d+' | head -1 || echo "0") +MIN_START=$((LAST_FINALIZED + 1)) +if [[ "$EXISTING_START" -gt "$MIN_START" ]]; then + START_BATCH="$EXISTING_START" +else + START_BATCH="$MIN_START" +fi + +log_info "Anvil RPC: $ANVIL_RPC" +log_info "MVRV: $MVRV" +log_info "Owner: $OWNER" +log_info "Min start: $MIN_START" +log_info "Existing start: $EXISTING_START" +log_info "Using start: $START_BATCH" +log_info "Bundle index: $DB_BUNDLE_INDEX" + +# --------------------------------------------------------------------------- +# Pre-flight: unlock owner for all impersonated transactions +# --------------------------------------------------------------------------- +cast rpc anvil_impersonateAccount "$OWNER" --rpc-url "$ANVIL_RPC" >/dev/null 2>&1 || true + +# --------------------------------------------------------------------------- +# 1. Deploy new Plonk Verifier from assets_v2/verifier.bin +# --------------------------------------------------------------------------- +PLONK_VERIFIER="" +if $deploy_plonk; then + VERIFIER_BIN="${ASSETS_DIR}/verifier.bin" + if [[ ! -f "$VERIFIER_BIN" ]]; then + log_error "Plonk verifier binary not found: $VERIFIER_BIN" + log_error "Make sure coordinator assets are downloaded (run coordinator once or download from S3)." + exit 1 + fi + + PLONK_BYTECODE=$(xxd -p "$VERIFIER_BIN" | tr -d '\n') + log_info "Deploying plonk verifier from $VERIFIER_BIN ($((${#PLONK_BYTECODE} / 2)) bytes) ..." + + # Anvil allows impersonation of any address with --unlocked + PLONK_DEPLOY_OUTPUT=$(cast send --rpc-url "$ANVIL_RPC" --chain 1 \ + --from "$OWNER" --unlocked --create "$PLONK_BYTECODE" 2>&1) + + # Extract deployed contract address from output + PLONK_VERIFIER=$(echo "$PLONK_DEPLOY_OUTPUT" | grep -oP 'contractAddress\s+\K0x[a-fA-F0-9]{40}' || true) + + if [[ -z "$PLONK_VERIFIER" ]]; then + log_error "Failed to extract plonk verifier address from cast output. Raw output:" + echo "$PLONK_DEPLOY_OUTPUT" + exit 1 + fi + + log_info "Plonk verifier deployed at: $PLONK_VERIFIER" +else + # If skipping plonk deploy, read existing deployed_verifier and extract its plonkVerifier + EXISTING_WRAPPER=$(jq -r '.contracts.deployed_verifier // empty' "$CONFIG_FILE") + if [[ -n "$EXISTING_WRAPPER" && "$EXISTING_WRAPPER" != "null" ]]; then + PLONK_VERIFIER=$(cast call "$EXISTING_WRAPPER" "plonkVerifier()(address)" --rpc-url "$ANVIL_RPC" 2>/dev/null || true) + log_info "Reusing plonk verifier from existing wrapper: $PLONK_VERIFIER" + fi + if [[ -z "$PLONK_VERIFIER" ]]; then + log_error "Cannot determine plonk verifier address. Either deploy one or provide an existing wrapper." + exit 1 + fi +fi + +# --------------------------------------------------------------------------- +# 2. Extract digests from DB proof instances +# --------------------------------------------------------------------------- +DIGEST1="" +DIGEST2="" +if $extract_digests; then + log_info "Extracting digests from bundle $DB_BUNDLE_INDEX proof instances ..." + + PROOF_JSON=$(psql "$DB_DSN" -Atq -c " + SELECT encode(proof, 'escape') + FROM bundle + WHERE index = $DB_BUNDLE_INDEX; + " 2>/dev/null) + + if [[ -z "$PROOF_JSON" ]]; then + log_error "Bundle $DB_BUNDLE_INDEX not found in DB" + exit 1 + fi + + # Parse instances base64 and extract digests + DIGESTS=$(echo "$PROOF_JSON" | python3 -c " +import sys, json, base64 +data = sys.stdin.read() +j = json.loads(data) +instances_raw = base64.b64decode(j['proof']['instances']) +# instances: 12 accumulators (384) + digest1 (32) + digest2 (32) + publicInputHash bytes (32*32=1024) +digest1 = '0x' + instances_raw[384:416].hex() +digest2 = '0x' + instances_raw[416:448].hex() +print(digest1) +print(digest2) +") + + DIGEST1=$(echo "$DIGESTS" | sed -n '1p') + DIGEST2=$(echo "$DIGESTS" | sed -n '2p') + + log_info "Extracted digest1: $DIGEST1" + log_info "Extracted digest2: $DIGEST2" +else + log_error "--skip-digests not supported; digests must always be extracted from proof" + exit 1 +fi + +# --------------------------------------------------------------------------- +# 3. Deploy ZkEvmVerifierPostFeynman wrapper +# --------------------------------------------------------------------------- +WRAPPER_ADDR="" +if $deploy_wrapper; then + # IMPORTANT: For new guest proofs (v0.8.0+), the correct wrapper is + # ZkEvmVerifierPostFeynman, NOT ZkEvmVerifierPostEuclid. + # PostFeynman computes keccak256(abi.encodePacked(protocolVersion, publicInput)) + # which matches the bundle_pi_hash embedded in the proof instances. + # protocolVersion = (domain << 6) + stf_version = (0 << 6) + 10 = 10 for Scroll+V10. + PROTOCOL_VERSION=10 + log_info "Deploying ZkEvmVerifierPostFeynman ..." + log_info " plonkVerifier: $PLONK_VERIFIER" + log_info " digest1: $DIGEST1" + log_info " digest2: $DIGEST2" + log_info " protocolVersion: $PROTOCOL_VERSION" + + cd "${PROJECT_ROOT}/../../scroll-contracts" + + WRAPPER_OUTPUT=$(forge create --broadcast --evm-version cancun --rpc-url "$ANVIL_RPC" \ + --from "$OWNER" --unlocked \ + src/libraries/verifier/ZkEvmVerifierPostFeynman.sol:ZkEvmVerifierPostFeynman \ + --constructor-args "$PLONK_VERIFIER" "$DIGEST1" "$DIGEST2" "$PROTOCOL_VERSION" 2>&1) + + WRAPPER_ADDR=$(echo "$WRAPPER_OUTPUT" | grep -oP 'Deployed to:\s+\K0x[a-fA-F0-9]{40}' || true) + + if [[ -z "$WRAPPER_ADDR" ]]; then + log_error "Failed to extract wrapper address from forge output. Raw output:" + echo "$WRAPPER_OUTPUT" + exit 1 + fi + + log_info "ZkEvmVerifierPostFeynman deployed at: $WRAPPER_ADDR" + + # Verify on-chain + ONCHAIN_DIGEST1=$(cast call "$WRAPPER_ADDR" "verifierDigest1()(bytes32)" --rpc-url "$ANVIL_RPC") + ONCHAIN_DIGEST2=$(cast call "$WRAPPER_ADDR" "verifierDigest2()(bytes32)" --rpc-url "$ANVIL_RPC") + ONCHAIN_PLONK=$(cast call "$WRAPPER_ADDR" "plonkVerifier()(address)" --rpc-url "$ANVIL_RPC") + + ONCHAIN_PROTO=$(cast call "$WRAPPER_ADDR" "protocolVersion()(uint256)" --rpc-url "$ANVIL_RPC") + log_info "On-chain verification:" + log_info " plonkVerifier: $ONCHAIN_PLONK" + log_info " digest1: $ONCHAIN_DIGEST1" + log_info " digest2: $ONCHAIN_DIGEST2" + log_info " protocolVersion: $ONCHAIN_PROTO" +else + WRAPPER_ADDR=$(jq -r '.contracts.deployed_verifier // empty' "$CONFIG_FILE") + log_info "Reusing existing wrapper: $WRAPPER_ADDR" +fi + +# --------------------------------------------------------------------------- +# 4. Register on MultipleVersionRollupVerifier +# --------------------------------------------------------------------------- +if $register; then + log_info "Registering verifier on MultipleVersionRollupVerifier ..." + log_info " version: 10" + log_info " startBatch: $START_BATCH" + log_info " verifier: $WRAPPER_ADDR" + + cast send "$MVRV" \ + "updateVerifier(uint256,uint64,address)" \ + 10 "$START_BATCH" "$WRAPPER_ADDR" \ + --from "$OWNER" --rpc-url "$ANVIL_RPC" --unlocked + + # Verify registration + REGISTERED=$(cast call "$MVRV" "getVerifier(uint256,uint256)(address)" 10 "$START_BATCH" --rpc-url "$ANVIL_RPC") + log_info "getVerifier(10, $START_BATCH) = $REGISTERED" + + if [[ "${REGISTERED,,}" != "${WRAPPER_ADDR,,}" ]]; then + log_error "Registration verification failed!" + exit 1 + fi + + log_info "Registration verified ✅" +fi + +# --------------------------------------------------------------------------- +# 5. Update config file +# --------------------------------------------------------------------------- +log_info "Updating config file: $CONFIG_FILE" +tmp=$(mktemp) +jq --arg addr "$WRAPPER_ADDR" '.contracts.deployed_verifier = $addr' "$CONFIG_FILE" > "$tmp" && mv "$tmp" "$CONFIG_FILE" +log_info "Config updated with deployed_verifier = $WRAPPER_ADDR" + +log_info "Done! 🎉" +log_info "" +log_info "Summary:" +log_info " Plonk Verifier: $PLONK_VERIFIER" +log_info " Wrapper: $WRAPPER_ADDR" +log_info " Digest1: $DIGEST1" +log_info " Digest2: $DIGEST2" diff --git a/tests/shadow-testing/scripts/04-prover-up.sh b/tests/shadow-testing/scripts/04-prover-up.sh new file mode 100755 index 0000000000..733e50c04f --- /dev/null +++ b/tests/shadow-testing/scripts/04-prover-up.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# Start prover(s) for shadow testing +# Usage: ./04-prover-up.sh --config mainnet --bundle-range 17297:17301 [--gpus 0,1] + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/lib/anvil-utils.sh" + +CONFIG="${CONFIG:-mainnet}" +GPUS="${GPUS:-0,1}" +DOCKER=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --config) CONFIG="$2"; shift 2 ;; + --gpus) GPUS="$2"; shift 2 ;; + --docker) DOCKER=true; shift ;; + -h|--help) sed -n '2,5p' "$0"; exit 0 ;; + *) log_error "Unknown option: $1"; exit 1 ;; + esac +done + +CONFIG_FILE="${SCRIPT_DIR}/../configs/${CONFIG}.json" +if [[ ! -f "$CONFIG_FILE" ]]; then + log_error "Config not found: $CONFIG_FILE" + exit 1 +fi + +# Read config +PROVER_NAME=$(jq -r '.prover.name_prefix' "$CONFIG_FILE") +CIRCUIT_VERSION=$(jq -r '.prover.circuit_version' "$CONFIG_FILE") +S3_URL=$(jq -r '.prover.s3_base_url' "$CONFIG_FILE") +DB_DSN=$(jq -r '.db.dsn' "$CONFIG_FILE") + +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" + +# ─── Build prover if needed ────────────────────────────────────────────────── +PROVER_BIN="${REPO_ROOT}/target/release/prover" + +if [[ ! -f "$PROVER_BIN" ]]; then + log_info "Building prover (GPU)..." + cd "${REPO_ROOT}/zkvm-prover" + make prover + log_ok " Built: $PROVER_BIN" +fi + +# ─── Launch provers ────────────────────────────────────────────────────────── +IFS=',' read -ra GPU_ARRAY <<< "$GPUS" + +for i in "${!GPU_ARRAY[@]}"; do + gpu_id="${GPU_ARRAY[$i]}" + prover_name="${PROVER_NAME}-${gpu_id}" + work_dir="${SCRIPT_DIR}/../.work/prover-${gpu_id}" + config_file="${work_dir}/prover.json" + log_file="${work_dir}/prover.log" + + mkdir -p "$work_dir" + + # Generate per-GPU config + cat > "$config_file" </dev/null || true + + log_info "Starting prover on GPU $gpu_id..." + + export RUST_MIN_STACK=16777216 + CUDA_VISIBLE_DEVICES="$gpu_id" nohup "$PROVER_BIN" \ + --config "$config_file" \ + > "$log_file" 2>&1 & + + pid=$! + echo "$pid" > "${work_dir}/prover.pid" + log_ok " Prover $gpu_id started (PID $pid, health :$((10080 + gpu_id)))" +done + +log_ok "All provers launched" diff --git a/tests/shadow-testing/scripts/05-wait-for-proofs.sh b/tests/shadow-testing/scripts/05-wait-for-proofs.sh new file mode 100755 index 0000000000..3c5ac3f4df --- /dev/null +++ b/tests/shadow-testing/scripts/05-wait-for-proofs.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Wait for bundle proofs to reach proving_status=4 (verified) +# Usage: ./05-wait-for-proofs.sh --bundle-range 17297:17301 [--timeout 3600] + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/lib/anvil-utils.sh" + +DB_DSN="${DB_DSN:-postgresql://postgres:shadow_pass@localhost:5433/shadow_rollup}" +BUNDLE_RANGE="" +TIMEOUT="${TIMEOUT:-7200}" # Default 2 hours +INTERVAL="${INTERVAL:-30}" # Check every 30s + +while [[ $# -gt 0 ]]; do + case "$1" in + --db-dsn) DB_DSN="$2"; shift 2 ;; + --bundle-range) BUNDLE_RANGE="$2"; shift 2 ;; + --timeout) TIMEOUT="$2"; shift 2 ;; + --interval) INTERVAL="$2"; shift 2 ;; + -h|--help) sed -n '2,5p' "$0"; exit 0 ;; + *) log_error "Unknown option: $1"; exit 1 ;; + esac +done + +if [[ -z "$BUNDLE_RANGE" ]]; then + log_error "Must specify --bundle-range" + exit 1 +fi + +b_start="${BUNDLE_RANGE%%:*}" +b_end="${BUNDLE_RANGE##*:}" +total=$((b_end - b_start + 1)) + +log_info "Waiting for $total bundles to be proven..." +log_info " Range: $BUNDLE_RANGE" +log_info " Timeout: ${TIMEOUT}s" + +start_time=$(date +%s) + +while true; do + result=$(psql "$DB_DSN" -Atq -c " + SELECT COUNT(*) FROM bundle + WHERE index BETWEEN $b_start AND $b_end + AND proving_status = 4 + " 2>/dev/null) + + done_count=$(echo "$result" | tr -d '\n') + elapsed=$(($(date +%s) - start_time)) + + printf "\r ⏳ %d/%d proven (%ds elapsed)" "$done_count" "$total" "$elapsed" + + if [[ "$done_count" -eq "$total" ]]; then + echo "" + log_ok "All $total bundles proven!" + break + fi + + if [[ "$elapsed" -ge "$TIMEOUT" ]]; then + echo "" + log_error "Timeout after ${TIMEOUT}s — only $done_count/$total proven" + exit 1 + fi + + sleep "$INTERVAL" +done diff --git a/tests/shadow-testing/scripts/06-run-relayer.sh b/tests/shadow-testing/scripts/06-run-relayer.sh new file mode 100755 index 0000000000..1181dec16d --- /dev/null +++ b/tests/shadow-testing/scripts/06-run-relayer.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# Build and launch the rollup relayer +# Usage: ./06-run-relayer.sh --config mainnet [--build] + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/lib/anvil-utils.sh" + +CONFIG="${CONFIG:-mainnet}" +BUILD=false +DOCKER=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --config) CONFIG="$2"; shift 2 ;; + --build) BUILD=true; shift ;; + --docker) DOCKER=true; shift ;; + -h|--help) sed -n '2,5p' "$0"; exit 0 ;; + *) log_error "Unknown option: $1"; exit 1 ;; + esac +done + +CONFIG_FILE="${SCRIPT_DIR}/../configs/${CONFIG}.json" +if [[ ! -f "$CONFIG_FILE" ]]; then + log_error "Config not found: $CONFIG_FILE" + exit 1 +fi + +# Read values from config +ANVIL_RPC=$(jq -r '.fork.anvil_rpc' "$CONFIG_FILE") +DB_DSN=$(jq -r '.db.dsn' "$CONFIG_FILE") +SCROLL_CHAIN=$(jq -r '.contracts.scroll_chain' "$CONFIG_FILE") +L2_ENDPOINT=$(jq -r '.e2e.l2_rpc' "$CONFIG_FILE") +VALIDIUM_MODE=$(jq -r '.relayer.validium_mode' "$CONFIG_FILE") +MIN_CODEC=$(jq -r '.relayer.min_codec_version' "$CONFIG_FILE") +CHAIN_MONITOR=$(jq -r '.relayer.chain_monitor_enabled' "$CONFIG_FILE") +GENESIS=$(jq -r '.genesis' "$CONFIG_FILE") + +# Use hardcoded dev keys for shadow testing +# Anvil default account #0 (commit) and the prover/finalize EOA +COMMIT_KEY="0x0afd95b5f1d9ef456b33c4e3720fbe70de7b4ff6e868fef454dc0aa60b09d8dc" +FINALIZE_KEY="0x01f1e12ee33f91d63172c3d51baa3cecb4469284b0ab45eed48e57fb5329ac4d" + +# ─── Render config ─────────────────────────────────────────────────────────── +RELAYER_CONFIG="${SCRIPT_DIR}/../.work/relayer-${CONFIG}.json" +mkdir -p "$(dirname "$RELAYER_CONFIG")" + +log_info "Rendering relayer config..." + +# Simple envsubst-style rendering +export ANVIL_RPC DB_DSN SCROLL_CHAIN L2_ENDPOINT COMMIT_KEY FINALIZE_KEY +export VALIDIUM_MODE CHAIN_MONITOR + +sed \ + -e "s|{{ANVIL_RPC}}|$ANVIL_RPC|g" \ + -e "s|{{DB_DSN}}|$DB_DSN|g" \ + -e "s|{{SCROLL_CHAIN}}|$SCROLL_CHAIN|g" \ + -e "s|{{L2_ENDPOINT}}|$L2_ENDPOINT|g" \ + -e "s|{{COMMIT_KEY}}|$COMMIT_KEY|g" \ + -e "s|{{FINALIZE_KEY}}|$FINALIZE_KEY|g" \ + -e "s|{{VALIDIUM_MODE}}|$VALIDIUM_MODE|g" \ + -e "s|{{CHAIN_MONITOR_ENABLED}}|$CHAIN_MONITOR|g" \ + "${SCRIPT_DIR}/../configs/relayer.json.template" > "$RELAYER_CONFIG" + +log_ok " Config written to $RELAYER_CONFIG" + +# ─── Build relayer ──────────────────────────────────────────────────────────── +# Resolve genesis path relative to repo root +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" + +RELAYER_BIN="${REPO_ROOT}/rollup/build/bin/rollup_relayer" + +if [[ "$BUILD" == "true" || ! -f "$RELAYER_BIN" ]]; then + log_info "Building relayer..." + cd "${REPO_ROOT}/rollup" + go build -o build/bin/rollup_relayer ./cmd/rollup_relayer + log_ok " Built: $RELAYER_BIN" +fi +if [[ ! "$GENESIS" = /* ]]; then + GENESIS="$REPO_ROOT/$GENESIS" +fi + +# ─── Launch relayer ─────────────────────────────────────────────────────────── +log_info "Starting relayer..." + +if [[ "$DOCKER" == "true" ]]; then + log_warn "Docker mode not yet implemented, falling back to bare metal" +fi + +# Kill any existing relayer +pkill -f "rollup_relayer.*relayer-${CONFIG}" 2>/dev/null || true +sleep 1 + +nohup "$RELAYER_BIN" \ + --config "$RELAYER_CONFIG" \ + --genesis "$GENESIS" \ + --min-codec-version "$MIN_CODEC" \ + --verbosity 3 \ + > "${SCRIPT_DIR}/../.work/relayer-${CONFIG}.log" 2>&1 & + +RELAYER_PID=$! +echo "$RELAYER_PID" > "${SCRIPT_DIR}/../.work/relayer-${CONFIG}.pid" + +log_ok " Relayer started (PID $RELAYER_PID)" +log_info " Logs: ${SCRIPT_DIR}/../.work/relayer-${CONFIG}.log" + +# Give it a moment to start +sleep 3 + +# Quick health check +if ! kill -0 "$RELAYER_PID" 2>/dev/null; then + log_error "Relayer exited immediately! Check logs." + tail -20 "${SCRIPT_DIR}/../.work/relayer-${CONFIG}.log" + exit 1 +fi + +log_ok "Relayer is running" diff --git a/tests/shadow-testing/scripts/07-wait-for-finalize.sh b/tests/shadow-testing/scripts/07-wait-for-finalize.sh new file mode 100755 index 0000000000..55ec75b93f --- /dev/null +++ b/tests/shadow-testing/scripts/07-wait-for-finalize.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# Wait for on-chain finalization of bundles +# Usage: ./07-wait-for-finalize.sh --bundle-range 17297:17301 --anvil-rpc ... --scroll-chain ... + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/lib/anvil-utils.sh" + +ANVIL_RPC="${ANVIL_RPC:-http://localhost:18545}" +SCROLL_CHAIN="${SCROLL_CHAIN:-0xa13BAF47339d63B743e7Da8741db5456DAc1E556}" +DB_DSN="${DB_DSN:-postgresql://postgres:shadow_pass@localhost:5433/shadow_rollup}" +BUNDLE_RANGE="" +TIMEOUT="${TIMEOUT:-1800}" # 30 min default +INTERVAL="${INTERVAL:-15}" +VERIFY_ONLY=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --anvil-rpc) ANVIL_RPC="$2"; shift 2 ;; + --scroll-chain) SCROLL_CHAIN="$2"; shift 2 ;; + --db-dsn) DB_DSN="$2"; shift 2 ;; + --bundle-range) BUNDLE_RANGE="$2"; shift 2 ;; + --timeout) TIMEOUT="$2"; shift 2 ;; + --interval) INTERVAL="$2"; shift 2 ;; + --verify-only) VERIFY_ONLY=true; shift ;; + -h|--help) sed -n '2,5p' "$0"; exit 0 ;; + *) log_error "Unknown option: $1"; exit 1 ;; + esac +done + +if [[ -z "$BUNDLE_RANGE" ]]; then + log_error "Must specify --bundle-range" + exit 1 +fi + +b_start="${BUNDLE_RANGE%%:*}" +b_end="${BUNDLE_RANGE##*:}" + +# We need to map bundle index to batch index. +# Query the DB to get the actual end_batch_index for the last bundle. +last_batch=$(psql "$DB_DSN" -Atq -c " + SELECT end_batch_index FROM bundle WHERE index = $b_end +" 2>/dev/null | tr -d '\n') + +if [[ -z "$last_batch" || "$last_batch" == "NULL" ]]; then + log_warn "Could not resolve batch index for bundle $b_end, falling back to bundle index" + last_batch="$b_end" +fi + +if [[ "$VERIFY_ONLY" == "true" ]]; then + current=$(cast call "$SCROLL_CHAIN" "lastFinalizedBatchIndex()(uint256)" --rpc-url "$ANVIL_RPC" 2>/dev/null | awk '{print $1}') + log_info "Current lastFinalizedBatchIndex: $current" + log_info "Target batch index: $last_batch" + if [[ "$current" -ge "$last_batch" ]]; then + log_ok "All bundles finalized!" + exit 0 + else + log_error "Not yet finalized ($current < $last_batch)" + exit 1 + fi +fi + +log_info "Waiting for finalization..." +log_info " Target batch: $last_batch" +log_info " Timeout: ${TIMEOUT}s" + +start_time=$(date +%s) + +while true; do + current=$(cast call "$SCROLL_CHAIN" "lastFinalizedBatchIndex()(uint256)" --rpc-url "$ANVIL_RPC" 2>/dev/null | awk '{print $1}' || echo "0") + elapsed=$(($(date +%s) - start_time)) + + printf "\r ⏳ lastFinalizedBatchIndex = %s / %s (%ds elapsed)" "$current" "$last_batch" "$elapsed" + + if [[ "$current" -ge "$last_batch" ]]; then + echo "" + log_ok "Finalization complete! lastFinalizedBatchIndex = $current" + break + fi + + if [[ "$elapsed" -ge "$TIMEOUT" ]]; then + echo "" + log_error "Timeout after ${TIMEOUT}s — lastFinalizedBatchIndex = $current" + exit 1 + fi + + sleep "$INTERVAL" +done diff --git a/tests/shadow-testing/scripts/08-docker-orchestrate.sh b/tests/shadow-testing/scripts/08-docker-orchestrate.sh new file mode 100755 index 0000000000..19a11318b5 --- /dev/null +++ b/tests/shadow-testing/scripts/08-docker-orchestrate.sh @@ -0,0 +1,359 @@ +#!/bin/bash +# One-command orchestrator for shadow fork testing. +# Usage: ./08-docker-orchestrate.sh [options] +# +# Options: +# --bundle-range RANGE Bundle index range, e.g. 17302:17305 +# --config NAME Config name (mainnet|sepolia), default: mainnet +# --phase PHASE Phase to run: env|prove|finalize|all (default: all) +# --skip-anvil-setup Skip Anvil state setup (use existing state) +# -h, --help Show this help + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" +source "${SCRIPT_DIR}/lib/anvil-utils.sh" + +# ─── Defaults ──────────────────────────────────────────────────────────────── +CONFIG="${CONFIG:-mainnet}" +export CONFIG +BUNDLE_RANGE="" +PHASE="all" +SKIP_ANVIL_SETUP=false + +# ─── Parse args ────────────────────────────────────────────────────────────── +while [[ $# -gt 0 ]]; do + case "$1" in + --bundle-range) BUNDLE_RANGE="$2"; shift 2 ;; + --config) CONFIG="$2"; shift 2 ;; + --phase) PHASE="$2"; shift 2 ;; + --skip-anvil-setup) SKIP_ANVIL_SETUP=true; shift ;; + -h|--help) + sed -n '2,14p' "$0" + exit 0 + ;; + *) log_error "Unknown option: $1"; exit 1 ;; + esac +done + +if [[ -z "$BUNDLE_RANGE" ]]; then + log_error "Must specify --bundle-range (e.g., 17302:17305)" + exit 1 +fi + +BUNDLE_START="${BUNDLE_RANGE%%:*}" +BUNDLE_END="${BUNDLE_RANGE##*:}" + +CONFIG_FILE="${SCRIPT_DIR}/../configs/${CONFIG}.json" +if [[ ! -f "$CONFIG_FILE" ]]; then + log_error "Config file not found: $CONFIG_FILE" + exit 1 +fi + +# ─── Read config ───────────────────────────────────────────────────────────── +log_info "Loading config: $CONFIG" + +FORK_URL=$(jq -r '.fork.url' "$CONFIG_FILE") +FORK_BLOCK=$(jq -r '.fork.block_number' "$CONFIG_FILE") +ANVIL_RPC=$(jq -r '.fork.anvil_rpc' "$CONFIG_FILE") +DB_DSN=$(jq -r '.db.dsn' "$CONFIG_FILE") +PROD_DSN="${PROD_DSN:-postgresql://mainnet_infra_team_read_only:AuexDUuaarskbG6tr9CH9gXsJqp4at67mddAbMrt@localhost:15432/mainnet_rollup}" +SCROLL_CHAIN=$(jq -r '.contracts.scroll_chain' "$CONFIG_FILE") +L1_MSG_QUEUE=$(jq -r '.contracts.l1_message_queue_v2' "$CONFIG_FILE") +ROLLUP_VERIF=$(jq -r '.contracts.rollup_verifier' "$CONFIG_FILE") +DEPLOYED_VERIF=$(jq -r '.contracts.deployed_verifier' "$CONFIG_FILE") +OWNER=$(jq -r '.contracts.owner' "$CONFIG_FILE") +PROVER_EOA=$(jq -r '.accounts.prover_eoa' "$CONFIG_FILE") +COMMIT_EOA=$(jq -r '.accounts.commit_eoa' "$CONFIG_FILE") +LAST_FINALIZED=$(jq -r '.reset.last_finalized_batch_index' "$CONFIG_FILE") +NEXT_QUEUE=$(jq -r '.reset.next_unfinalized_queue_index' "$CONFIG_FILE") +CODEC_VERSION=$(jq -r '.reset.codec_version' "$CONFIG_FILE") +GENESIS=$(jq -r '.genesis' "$CONFIG_FILE") +L2_RPC=$(jq -r '.e2e.l2_rpc // .coordinator.l2geth // "https://mainnet-rpc.scroll.io"' "$CONFIG_FILE") +VALIDIUM_MODE=$(jq -r '.relayer.validium_mode // false' "$CONFIG_FILE") +CHAIN_MONITOR=$(jq -r '.relayer.chain_monitor_enabled // false' "$CONFIG_FILE") + +# Docker compose file (run from repo root) +COMPOSE_FILE="${REPO_ROOT}/tests/shadow-testing/docker-compose.yml" + +# ─── Helpers ───────────────────────────────────────────────────────────────── + +compose() { + docker compose -f "$COMPOSE_FILE" "$@" +} + +wait_for_postgres() { + local max=30 + local i=1 + while [ $i -le $max ]; do + if compose exec -T postgres pg_isready -U postgres >/dev/null 2>&1; then + log_ok "PostgreSQL is ready" + return 0 + fi + log_info "Waiting for PostgreSQL... ($i/$max)" + sleep 2 + ((i++)) + done + log_error "PostgreSQL failed to start" + return 1 +} + +wait_for_coordinator() { + local max=60 + local i=1 + while [ $i -le $max ]; do + if curl -s http://localhost:8390/ >/dev/null 2>&1; then + log_ok "Coordinator is ready at :8390" + return 0 + fi + log_info "Waiting for coordinator... ($i/$max)" + sleep 5 + ((i++)) + done + log_error "Coordinator failed to start" + return 1 +} + +wait_for_prover() { + local max=120 + local i=1 + while [ $i -le $max ]; do + # Check prover health via docker exec (no port mapping needed) + if docker exec shadow-prover-gpu-0 sh -c "curl -sf http://localhost:10080/health >/dev/null 2>&1" 2>/dev/null; then + log_ok "Prover is ready at :10080" + return 0 + fi + # Fallback: check if container is still running + if ! docker ps --format '{{.Names}}' | grep -q "^shadow-prover-gpu-0$"; then + log_error "Prover container exited unexpectedly" + return 1 + fi + log_info "Waiting for prover... ($i/$max)" + sleep 10 + ((i++)) + done + log_error "Prover failed to start" + return 1 +} + +render_relayer_config() { + local output="${SCRIPT_DIR}/../.work/relayer-${CONFIG}.json" + mkdir -p "$(dirname "$output")" + + local template="${SCRIPT_DIR}/../configs/relayer.json.template" + sed \ + -e "s|{{ANVIL_RPC}}|$ANVIL_RPC|g" \ + -e "s|{{DB_DSN}}|$DB_DSN|g" \ + -e "s|{{SCROLL_CHAIN}}|$SCROLL_CHAIN|g" \ + -e "s|{{L2_ENDPOINT}}|$L2_RPC|g" \ + -e "s|{{COMMIT_KEY}}|0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80|g" \ + -e "s|{{FINALIZE_KEY}}|0x01f1e12ee33f91d63172c3d51baa3cecb4469284b0ab45eed48e57fb5329ac4d|g" \ + -e "s|{{VALIDIUM_MODE}}|$VALIDIUM_MODE|g" \ + -e "s|{{CHAIN_MONITOR_ENABLED}}|$CHAIN_MONITOR|g" \ + "$template" > "$output" + log_ok "Relayer config rendered: $output" +} + +render_prover_config() { + local gpu_id="${1:-0}" + local output="${SCRIPT_DIR}/../.work/prover-${gpu_id}.json" + mkdir -p "$(dirname "$output")" + + local prover_name + prover_name=$(jq -r '.prover.name_prefix' "$CONFIG_FILE") + local s3_url + s3_url=$(jq -r '.prover.s3_base_url' "$CONFIG_FILE") + + cat > "$output" </dev/null 2>&1 || { log_error "docker is required"; exit 1; } + command -v jq >/dev/null 2>&1 || { log_error "jq is required"; exit 1; } + command -v anvil >/dev/null 2>&1 || { log_error "anvil (Foundry) is required"; exit 1; } + + # 2. Start / reset PostgreSQL + log_info "Starting PostgreSQL..." + if compose ps postgres 2>/dev/null | grep -q "running"; then + log_info " PostgreSQL already running, stopping first..." + compose stop postgres >/dev/null 2>&1 || true + compose rm -f postgres >/dev/null 2>&1 || true + fi + compose up postgres -d + wait_for_postgres + + # 3. Import bundle range + log_info "Importing bundle range $BUNDLE_RANGE..." + "${SCRIPT_DIR}/00-import-bundle-range.sh" \ + --bundle-range "$BUNDLE_RANGE" \ + --prod-dsn "$PROD_DSN" \ + --shadow-dsn "$DB_DSN" + + # 4. Start Anvil (bare metal) + log_info "Starting Anvil fork..." + local anvil_port="${ANVIL_RPC##*:}" + local existing_pid + existing_pid=$(lsof -ti :"$anvil_port" 2>/dev/null || true) + if [[ -n "$existing_pid" ]]; then + log_warn "Killing existing Anvil on port $anvil_port (PID $existing_pid)" + kill "$existing_pid" 2>/dev/null || true + sleep 2 + fi + + local state_file="${SCRIPT_DIR}/../.work/anvil-${CONFIG}.state.json" + mkdir -p "$(dirname "$state_file")" + + nohup anvil \ + --fork-url "$FORK_URL" \ + --fork-block-number "$FORK_BLOCK" \ + --block-time 12 \ + --port "$anvil_port" \ + --host 0.0.0.0 \ + --state "$state_file" \ + >/dev/null 2>&1 & + ANVIL_PID=$! + log_info "Anvil started (PID $ANVIL_PID, port $anvil_port)" + echo "$ANVIL_PID" > "${SCRIPT_DIR}/../.work/anvil-${CONFIG}.pid" + sleep 3 + + # 5. Setup Anvil state + if [[ "$SKIP_ANVIL_SETUP" == "false" ]]; then + log_info "Setting up Anvil state..." + "${SCRIPT_DIR}/01-setup-anvil.sh" \ + --no-anvil \ + --anvil-rpc "$ANVIL_RPC" \ + --last-finalized "$LAST_FINALIZED" \ + --next-queue "$NEXT_QUEUE" \ + --deployed-verifier "$DEPLOYED_VERIF" \ + --prover-eoa "$PROVER_EOA" \ + --commit-eoa "$COMMIT_EOA" \ + --owner "$OWNER" \ + --scroll-chain "$SCROLL_CHAIN" \ + --l1-msg-queue "$L1_MSG_QUEUE" \ + --rollup-verifier "$ROLLUP_VERIF" \ + --db-dsn "$DB_DSN" \ + --codec-version "$CODEC_VERSION" + else + log_info "Skipping Anvil state setup (--skip-anvil-setup)" + fi + + log_ok "Phase env complete!" +} + +# ─── Phase: prove ──────────────────────────────────────────────────────────── + +run_prove() { + log_info "=== Phase: prove ===" + + # 6. Start coordinator + log_info "Starting coordinator..." + compose up coordinator -d + wait_for_coordinator + + # 7. Render prover config and start prover + log_info "Rendering prover config..." + render_prover_config 0 + log_info "Starting prover (GPU 0)..." + compose --profile coordinator --profile prover up prover-gpu-0 -d + wait_for_prover + + # 8. Wait for proofs + log_info "Waiting for proofs..." + "${SCRIPT_DIR}/05-wait-for-proofs.sh" \ + --db-dsn "$DB_DSN" \ + --bundle-range "$BUNDLE_RANGE" + + log_ok "Phase prove complete!" +} + +# ─── Phase: finalize ───────────────────────────────────────────────────────── + +run_finalize() { + log_info "=== Phase: finalize ===" + + # 9. Render relayer config + render_relayer_config + + # 10. Start relayer + log_info "Starting relayer..." + compose up relayer -d + sleep 3 + + # 11. Wait for finalization + log_info "Waiting for finalization..." + "${SCRIPT_DIR}/07-wait-for-finalize.sh" \ + --anvil-rpc "$ANVIL_RPC" \ + --scroll-chain "$SCROLL_CHAIN" \ + --bundle-range "$BUNDLE_RANGE" \ + --db-dsn "$DB_DSN" + + log_ok "Phase finalize complete!" +} + +# ─── Main ──────────────────────────────────────────────────────────────────── + +log_info "Shadow Fork Orchestrator" +log_info " Config: $CONFIG" +log_info " Bundles: $BUNDLE_RANGE" +log_info " Phase: $PHASE" + +# Ensure .work dir exists +mkdir -p "${SCRIPT_DIR}/../.work" + +case "$PHASE" in + env) + run_env + ;; + prove) + run_prove + ;; + finalize) + run_finalize + ;; + all) + run_env + run_prove + run_finalize + log_info "" + log_ok "🎉 Full pipeline complete!" + log_info " Config: $CONFIG" + log_info " Bundles: $BUNDLE_RANGE" + ;; + *) + log_error "Unknown phase: $PHASE (expected: env|prove|finalize|all)" + exit 1 + ;; +esac diff --git a/tests/shadow-testing/scripts/fetch-l2-blocks.py b/tests/shadow-testing/scripts/fetch-l2-blocks.py new file mode 100755 index 0000000000..9948de66ea --- /dev/null +++ b/tests/shadow-testing/scripts/fetch-l2-blocks.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +""" +Fetch L2 block headers from RPC and populate l2_block table in shadow DB. + +The coordinator needs l2_block records to format chunk tasks (for block hashes +and hardfork name resolution). This script fetches blocks in batches and +inserts them into the shadow database. + +Usage: + python3 fetch-l2-blocks.py --rpc https://mainnet-rpc.scroll.io \ + --db "postgresql://:@localhost:5433/shadow_rollup" \ + --start-block 26000000 --end-block 27000000 + +After running, link blocks to chunks: + UPDATE l2_block lb + SET chunk_hash = c.hash + FROM chunk c + WHERE lb.number >= c.start_block_number + AND lb.number <= c.end_block_number; +""" + +import argparse +import sys +import time +import concurrent.futures +from typing import Optional + +import requests +import psycopg2 +from psycopg2.extras import execute_values + + +def fetch_block_batch(rpc_url: str, block_numbers: list[int]) -> list[dict]: + """Fetch multiple blocks via batch JSON-RPC request.""" + payload = [ + { + "jsonrpc": "2.0", + "method": "eth_getBlockByNumber", + "params": [hex(num), False], + "id": i, + } + for i, num in enumerate(block_numbers) + ] + + try: + resp = requests.post(rpc_url, json=payload, headers={"Content-Type": "application/json"}, timeout=60) + resp.raise_for_status() + results = resp.json() + + blocks = [] + for result in results: + if "error" in result: + print(f" Error fetching block: {result['error']}", file=sys.stderr) + continue + block = result.get("result") + if block is None: + continue + blocks.append(block) + return blocks + except Exception as e: + print(f" Request failed: {e}", file=sys.stderr) + return [] + + +def insert_blocks(db_url: str, blocks: list[dict]) -> int: + """Insert blocks into l2_block table.""" + if not blocks: + return 0 + + rows = [] + for block in blocks: + try: + number = int(block["number"], 16) + hash_val = block["hash"] + parent_hash = block["parentHash"] + timestamp = int(block["timestamp"], 16) + gas_used = int(block["gasUsed"], 16) + rows.append((number, hash_val, parent_hash, timestamp, gas_used)) + except (KeyError, ValueError) as e: + print(f" Skipping malformed block: {e}", file=sys.stderr) + continue + + if not rows: + return 0 + + conn = psycopg2.connect(db_url) + try: + with conn.cursor() as cur: + execute_values( + cur, + """ + INSERT INTO l2_block (number, hash, parent_hash, timestamp, gas_used) + VALUES %s + ON CONFLICT (number) DO UPDATE SET + hash = EXCLUDED.hash, + parent_hash = EXCLUDED.parent_hash, + timestamp = EXCLUDED.timestamp, + gas_used = EXCLUDED.gas_used + """, + rows, + ) + conn.commit() + return len(rows) + finally: + conn.close() + + +def get_existing_block_range(db_url: str) -> tuple[Optional[int], Optional[int]]: + """Get min/max block numbers already in the DB.""" + conn = psycopg2.connect(db_url) + try: + with conn.cursor() as cur: + cur.execute("SELECT MIN(number), MAX(number) FROM l2_block") + return cur.fetchone() + finally: + conn.close() + + +def main(): + parser = argparse.ArgumentParser(description="Fetch L2 blocks into shadow DB") + parser.add_argument("--rpc", required=True, help="L2 RPC endpoint URL") + parser.add_argument("--db", required=True, help="Shadow DB connection string") + parser.add_argument("--start-block", type=int, required=True, help="First block to fetch") + parser.add_argument("--end-block", type=int, required=True, help="Last block to fetch") + parser.add_argument("--batch-size", type=int, default=100, help="RPC batch size (default: 100)") + parser.add_argument("--workers", type=int, default=4, help="Concurrent workers (default: 4)") + parser.add_argument("--delay", type=float, default=0.1, help="Delay between batches in seconds (default: 0.1)") + parser.add_argument("--skip-existing", action="store_true", help="Skip blocks already in DB") + args = parser.parse_args() + + existing_min, existing_max = get_existing_block_range(args.db) + print(f"Existing blocks in DB: {existing_min or 'none'} to {existing_max or 'none'}") + + start = args.start_block + end = args.end_block + + if args.skip_existing and existing_min is not None: + # Only fetch gaps or new blocks + # Simple approach: just fetch the requested range, ON CONFLICT will handle it + pass + + total_blocks = end - start + 1 + print(f"Fetching {total_blocks} blocks from {start} to {end} via {args.rpc}") + + fetched = 0 + failed = 0 + + # Generate batch ranges + ranges = [] + current = start + while current <= end: + batch_end = min(current + args.batch_size - 1, end) + ranges.append(list(range(current, batch_end + 1))) + current = batch_end + 1 + + with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as executor: + futures = { + executor.submit(fetch_block_batch, args.rpc, block_nums): block_nums + for block_nums in ranges + } + + for future in concurrent.futures.as_completed(futures): + block_nums = futures[future] + try: + blocks = future.result() + if blocks: + inserted = insert_blocks(args.db, blocks) + fetched += inserted + print(f" Blocks {block_nums[0]}-{block_nums[-1]}: inserted {inserted}/{len(blocks)}") + else: + failed += len(block_nums) + print(f" Blocks {block_nums[0]}-{block_nums[-1]}: FAILED") + except Exception as e: + failed += len(block_nums) + print(f" Blocks {block_nums[0]}-{block_nums[-1]}: ERROR {e}") + + time.sleep(args.delay) + + print(f"\nDone! Fetched: {fetched}, Failed: {failed}") + print("\nNext step: link blocks to chunks:") + print(""" + UPDATE l2_block lb + SET chunk_hash = c.hash + FROM chunk c + WHERE lb.number >= c.start_block_number + AND lb.number <= c.end_block_number; + """) + + +if __name__ == "__main__": + main() diff --git a/tests/shadow-testing/scripts/fix_l2_block_transactions.py b/tests/shadow-testing/scripts/fix_l2_block_transactions.py new file mode 100644 index 0000000000..c25f8e9582 --- /dev/null +++ b/tests/shadow-testing/scripts/fix_l2_block_transactions.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +""" +Fix missing l2_block.transactions data by fetching from L2 RPC. +""" + +import json +import psycopg2 +import requests +import sys +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Optional + +L2_RPC = "https://l2geth-rpc-proxy.mainnet.aws.scroll.io" +DB_DSN = "postgresql://postgres:shadow_pass@localhost:5433/shadow_rollup" +BATCH_SIZE = 50 +MAX_WORKERS = 20 + + +def hex_to_int(hex_str: str) -> int: + if hex_str is None: + return 0 + return int(hex_str, 16) + + +def rpc_get_block(block_num: int) -> Optional[dict]: + """Fetch full block with transactions from L2 RPC.""" + try: + resp = requests.post( + L2_RPC, + json={ + "jsonrpc": "2.0", + "method": "eth_getBlockByNumber", + "params": [hex(block_num), True], + "id": 1, + }, + timeout=30, + ) + resp.raise_for_status() + result = resp.json().get("result") + return result + except Exception as e: + print(f"Error fetching block {block_num}: {e}", file=sys.stderr) + return None + + +def convert_tx_to_transaction_data(rpc_tx: dict) -> dict: + """Convert RPC transaction JSON to TransactionData format.""" + tx_type = hex_to_int(rpc_tx.get("type", "0x0")) + + # Handle L1 message tx: nonce should be queueIndex, not tx.Nonce() + if tx_type == 0x7E: + nonce = hex_to_int(rpc_tx.get("queueIndex", "0x0")) + else: + nonce = hex_to_int(rpc_tx.get("nonce", "0x0")) + + # gasPrice: for legacy txs, use gasPrice; for EIP-1559, use effective gasPrice + gas_price = rpc_tx.get("gasPrice") + if gas_price is None: + gas_price = rpc_tx.get("maxFeePerGas", "0x0") + + # gasTipCap / gasFeeCap + gas_tip_cap = rpc_tx.get("maxPriorityFeePerGas", gas_price) + gas_fee_cap = rpc_tx.get("maxFeePerGas", gas_price) + + # isCreate: true if 'to' is null + to_addr = rpc_tx.get("to") + is_create = to_addr is None + + # accessList: use null if empty or absent + access_list = rpc_tx.get("accessList") + if access_list == []: + access_list = None + + # authorizationList + auth_list = rpc_tx.get("authorizationList") + if auth_list == []: + auth_list = None + + # v: use yParity if available for EIP-1559, otherwise use v + v = rpc_tx.get("v") + if v is None: + v = rpc_tx.get("yParity", "0x0") + + tx_data = { + "type": tx_type, + "nonce": nonce, + "txHash": rpc_tx.get("hash", ""), + "gas": hex_to_int(rpc_tx.get("gas", "0x0")), + "gasPrice": gas_price, + "gasTipCap": gas_tip_cap, + "gasFeeCap": gas_fee_cap, + "from": rpc_tx.get("from", ""), + "to": to_addr, + "chainId": rpc_tx.get("chainId", "0x82750"), + "value": rpc_tx.get("value", "0x0"), + "data": rpc_tx.get("input", "0x"), + "isCreate": is_create, + "accessList": access_list, + "authorizationList": auth_list, + "v": v, + "r": rpc_tx.get("r", "0x0"), + "s": rpc_tx.get("s", "0x0"), + } + + return tx_data + + +def process_block(block_num: int) -> Optional[tuple]: + """Fetch and convert transactions for a single block.""" + block = rpc_get_block(block_num) + if block is None: + return None + + rpc_txs = block.get("transactions", []) + if not rpc_txs: + tx_data = "[]" + else: + tx_list = [convert_tx_to_transaction_data(tx) for tx in rpc_txs] + tx_data = json.dumps(tx_list, separators=(',', ':')) + + return (block_num, tx_data) + + +def update_blocks_in_db(blocks_data: list): + """Update transactions for multiple blocks in shadow DB.""" + conn = psycopg2.connect(DB_DSN) + cur = conn.cursor() + + try: + for block_num, tx_data in blocks_data: + cur.execute( + "UPDATE l2_block SET transactions = %s WHERE number = %s", + (tx_data, block_num) + ) + conn.commit() + except Exception as e: + conn.rollback() + print(f"DB update error: {e}", file=sys.stderr) + raise + finally: + cur.close() + conn.close() + + +def get_empty_blocks() -> list: + """Get list of block numbers with empty transactions.""" + conn = psycopg2.connect(DB_DSN) + cur = conn.cursor() + cur.execute( + "SELECT number FROM l2_block WHERE transactions = '' ORDER BY number" + ) + blocks = [row[0] for row in cur.fetchall()] + cur.close() + conn.close() + return blocks + + +def main(): + empty_blocks = get_empty_blocks() + total = len(empty_blocks) + print(f"Found {total} blocks with empty transactions") + + if total == 0: + print("No empty transactions to fix.") + return + + processed = 0 + failed_blocks = [] + + for batch_start in range(0, total, BATCH_SIZE): + batch = empty_blocks[batch_start:batch_start + BATCH_SIZE] + print(f"Processing batch {batch_start//BATCH_SIZE + 1}/{(total-1)//BATCH_SIZE + 1}: blocks {batch[0]} to {batch[-1]}") + + blocks_data = [] + with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: + futures = {executor.submit(process_block, bn): bn for bn in batch} + for future in as_completed(futures): + result = future.result() + if result: + blocks_data.append(result) + else: + failed_blocks.append(futures[future]) + + if blocks_data: + update_blocks_in_db(blocks_data) + processed += len(blocks_data) + print(f" Updated {len(blocks_data)} blocks. Total processed: {processed}/{total}") + + # Small delay between batches to avoid rate limiting + time.sleep(0.5) + + print(f"\nDone! Processed {processed}/{total} blocks.") + if failed_blocks: + print(f"Failed blocks: {failed_blocks}") + + +if __name__ == "__main__": + main() diff --git a/tests/shadow-testing/scripts/lib/anvil-utils.sh b/tests/shadow-testing/scripts/lib/anvil-utils.sh new file mode 100755 index 0000000000..981e51c60c --- /dev/null +++ b/tests/shadow-testing/scripts/lib/anvil-utils.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# Shared Anvil utility functions for shadow testing +set -euo pipefail + +# Note: do not define SCRIPT_DIR here, as this file is sourced by other scripts +# and would overwrite their SCRIPT_DIR variable. + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +log_info() { echo -e "${BLUE}[INFO]${NC} $*"; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } +log_error() { echo -e "${RED}[ERROR]${NC} $*"; } +log_ok() { echo -e "${GREEN}[OK]${NC} $*"; } + +# Wait for Anvil to be ready +wait_for_anvil() { + local rpc_url="${1:-http://localhost:18545}" + local max_wait="${2:-60}" + log_info "Waiting for Anvil at $rpc_url ..." + for ((i=0; i/dev/null 2>&1; then + log_ok "Anvil is ready" + return 0 + fi + sleep 1 + done + log_error "Anvil did not become ready within ${max_wait}s" + return 1 +} + +# Get storage slot value +get_storage() { + local contract="$1" + local slot="$2" + local rpc_url="${3:-http://localhost:18545}" + cast storage "$contract" "$slot" --rpc-url "$rpc_url" 2>/dev/null | tr -d '\n' +} + +# Set storage slot value (Anvil only) +set_storage() { + local contract="$1" + local slot="$2" + local value="$3" + local rpc_url="${4:-http://localhost:18545}" + cast rpc anvil_setStorageAt "$contract" "$slot" "$value" --rpc-url "$rpc_url" >/dev/null +} + +# Impersonate an account (Anvil only) +impersonate() { + local addr="$1" + local rpc_url="${2:-http://localhost:18545}" + cast rpc anvil_impersonateAccount "$addr" --rpc-url "$rpc_url" >/dev/null +} + +# Stop impersonating +stop_impersonate() { + local addr="$1" + local rpc_url="${2:-http://localhost:18545}" + cast rpc anvil_stopImpersonatingAccount "$addr" --rpc-url "$rpc_url" >/dev/null +} + +# Send ETH to an address +set_balance() { + local addr="$1" + local wei="${2:-0x56bc75e2d63100000}" # 100 ETH default + local rpc_url="${3:-http://localhost:18545}" + cast rpc anvil_setBalance "$addr" "$wei" --rpc-url "$rpc_url" >/dev/null +} + +# Encode uint256 for storage +encode_uint256() { + local val="$1" + printf '%064x' "$val" +} + +# Extract lower 64 bits from a 32-byte hex string +lower64() { + local hex="$1" + echo "${hex: -16}" +} + +# Check if a command exists +require_cmd() { + if ! command -v "$1" &>/dev/null; then + log_error "Required command not found: $1" + exit 1 + fi +} + +# Parse bundle range like "17297:17301" into individual indices +parse_bundle_range() { + local range="$1" + local start="${range%%:*}" + local end="${range##*:}" + for ((i=start; i<=end; i++)); do + echo "$i" + done +}