Merge celestia v3.3.2

.github/buildspec.yml

@@ -0,0 +1,36 @@
+version: 0.2
+
+phases:
+  pre_build:
+    commands:
+      - git submodule update --init
+      - echo Logging in to Dockerhub....
+      - docker login --username $DOCKERHUB_USERNAME --password $DOCKERHUB_PASSWORD
+      - aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin $REPOSITORY_URI
+      - COMMIT_HASH=$(git rev-parse --short=7 HEAD || echo "latest")
+      - VERSION_TAG=$(git tag --points-at HEAD | sed '/-/!s/$/_/' | sort -rV | sed 's/_$//' | head -n 1 | grep ^ || git show -s --pretty=%D | sed 's/, /\n/g' | grep -v '^origin/' |grep -v '^grafted\|HEAD\|master\|main$' || echo "dev")
+      - NITRO_VERSION=${VERSION_TAG}-${COMMIT_HASH}
+      - IMAGE_TAG=${NITRO_VERSION}
+      - NITRO_DATETIME=$(git show -s --date=iso-strict --format=%cd)
+      - NITRO_MODIFIED="false"
+      - echo ${NITRO_VERSION} > ./.nitro-tag.txt
+  build:
+    commands:
+      - echo Build started on `date` 
+      - echo Building the Docker image ${NITRO_VERSION}...
+      - DOCKER_BUILDKIT=1 docker build . -t nitro-node-slim --target nitro-node-slim --build-arg version=$NITRO_VERSION --build-arg datetime=$NITRO_DATETIME --build-arg modified=$NITRO_MODIFIED
+      - DOCKER_BUILDKIT=1 docker build . -t nitro-node --target nitro-node --build-arg version=$NITRO_VERSION --build-arg datetime=$NITRO_DATETIME --build-arg modified=$NITRO_MODIFIED
+      - DOCKER_BUILDKIT=1 docker build . -t nitro-node-dev --target nitro-node-dev --build-arg version=$NITRO_VERSION --build-arg datetime=$NITRO_DATETIME --build-arg modified=$NITRO_MODIFIED
+      - DOCKER_BUILDKIT=1 docker build . -t nitro-node-validator --target nitro-node-validator --build-arg version=$NITRO_VERSION --build-arg datetime=$NITRO_DATETIME --build-arg modified=$NITRO_MODIFIED
+      - docker tag nitro-node:latest $REPOSITORY_URI:$IMAGE_TAG-$ARCH_TAG
+      - docker tag nitro-node-slim:latest $REPOSITORY_URI:$IMAGE_TAG-slim-$ARCH_TAG
+      - docker tag nitro-node-dev:latest $REPOSITORY_URI:$IMAGE_TAG-dev-$ARCH_TAG
+      - docker tag nitro-node-validator:latest $REPOSITORY_URI:$IMAGE_TAG-validator-$ARCH_TAG
+  post_build: 
+    commands:
+      - echo Build completed on `date` 
+      - echo pushing to repo
+      - docker push $REPOSITORY_URI:$IMAGE_TAG-$ARCH_TAG
+      - docker push $REPOSITORY_URI:$IMAGE_TAG-slim-$ARCH_TAG
+      - docker push $REPOSITORY_URI:$IMAGE_TAG-dev-$ARCH_TAG
+      - docker push $REPOSITORY_URI:$IMAGE_TAG-validator-$ARCH_TAG

.github/workflows/arbitrator-ci.yml

@@ -50,15 +50,13 @@ jobs:
       - name: Install go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.21.x
+          go-version: 1.23.x
 
       - name: Install custom go-ethereum
         run: |
           cd /tmp
-          git clone --branch v1.13.8 --depth 1 https://github.com/ethereum/go-ethereum.git
+          git clone --branch v1.14.11 --depth 1 https://github.com/ethereum/go-ethereum.git
           cd go-ethereum
-          # Enable KZG point evaluation precompile early
-          sed -i 's#var PrecompiledContractsBerlin = map\[common.Address\]PrecompiledContract{#\0 common.BytesToAddress([]byte{0x0a}): \&kzgPointEvaluation{},#g' core/vm/contracts.go
           go build -o /usr/local/bin/geth ./cmd/geth
 
       - name: Setup nodejs

.github/workflows/ci.yml

@@ -102,7 +102,7 @@ jobs:
       - name: Install go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.21.x
+          go-version: 1.23.x
 
       - name: Install wasm-ld
         run: |
@@ -147,14 +147,14 @@ jobs:
         uses: actions/cache@v3
         with:
           path: |
-            ~/.cargo/registry/
-            ~/.cargo/git/
-            arbitrator/**/target/
+            ~/.cargo/
+            arbitrator/target/
             arbitrator/wasm-libraries/target/
-            arbitrator/wasm-libraries/soft-float/SoftFloat/build
+            arbitrator/wasm-libraries/soft-float/
             target/etc/initial-machine-cache/
-          key: v2-${{ runner.os }}-cargo-${{ steps.install-rust.outputs.rustc_hash }}-min-${{ hashFiles('arbitrator/Cargo.lock') }}-${{ matrix.test-mode }}
-          restore-keys: v1-${{ runner.os }}-cargo-${{ steps.install-rust.outputs.rustc_hash }}-
+            /home/runner/.rustup/toolchains/
+          key: ${{ runner.os }}-cargo-${{ steps.install-rust.outputs.rustc_hash }}-min-${{ hashFiles('arbitrator/Cargo.lock') }}-${{ matrix.test-mode }}
+          restore-keys: ${{ runner.os }}-cargo-${{ steps.install-rust.outputs.rustc_hash }}-
 
       - name: Cache cbrotli
         uses: actions/cache@v3
@@ -189,14 +189,14 @@ jobs:
       - name: Build all lint dependencies
         run: make -j build-node-deps
 
-      - name: Lint
-        if: matrix.test-mode == 'defaults'
-        uses: golangci/golangci-lint-action@v5
-        with:
-          version: v1.59.1
-          skip-cache: true
+      # TODO: celestia has lint errors, fix in later PRs
+      # - name: Lint
+      #   if: matrix.test-mode == 'defaults'
+      #   uses: golangci/golangci-lint-action@v3
+      #   with:
+      #     version: v1.59.1
+      #     skip-cache: true
 
-      # TODO: celestia has lint errors, fix later
       # - name: Custom Lint
       #   if: matrix.test-mode == 'defaults'
       #   run: |

.github/workflows/codeql-analysis.yml

@@ -73,7 +73,7 @@ jobs:
     - name: Install go
       uses: actions/setup-go@v4
       with:
-        go-version: 1.21.x
+        go-version: 1.23.x
 
     - name: Install rust stable
       uses: dtolnay/rust-toolchain@stable

.github/workflows/gotestsum.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+
+check_missing_value() {
+    if [[ $1 -eq 0 || $2 == -* ]]; then
+        echo "missing $3 argument value"
+        exit 1
+    fi
+}
+
+timeout=""
+tags=""
+run=""
+race=false
+cover=false
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --timeout)
+      shift
+      check_missing_value $# "$1" "--timeout"
+      timeout=$1
+      shift
+      ;;
+    --tags)
+      shift
+      check_missing_value $# "$1" "--tags"
+      tags=$1
+      shift
+      ;;
+    --run)
+      shift
+      check_missing_value $# "$1" "--run"
+      run=$1
+      shift
+      ;;
+    --race)
+      race=true
+      shift
+      ;;
+    --cover)
+      cover=true
+      shift
+      ;;
+    *)
+      echo "Invalid argument: $1"
+      exit 1
+      ;;
+  esac
+done
+
+packages=$(go list ./...)
+for package in $packages; do
+  cmd="stdbuf -oL gotestsum --format short-verbose --packages=\"$package\" --rerun-fails=2 --no-color=false --"
+
+  if [ "$timeout" != "" ]; then
+    cmd="$cmd -timeout $timeout"
+  fi
+
+  if [ "$tags" != "" ]; then
+    cmd="$cmd -tags=$tags"
+  fi
+
+  if [ "$run" != "" ]; then
+    cmd="$cmd -run=$run"
+  fi
+
+  if [ "$race" == true ]; then
+    cmd="$cmd -race"
+  fi
+
+  if [ "$cover" == true ]; then
+    cmd="$cmd -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/..."
+  fi
+
+  cmd="$cmd > >(stdbuf -oL tee -a full.log | grep -vE \"INFO|seal\")"
+
+  echo ""
+  echo running tests for "$package"
+  echo "$cmd"
+
+  if ! eval "$cmd"; then
+    exit 1
+  fi
+done

.github/workflows/submodule-pin-check.yml

@@ -18,7 +18,8 @@ jobs:
         with:
           fetch-depth: 0
           submodules: true
-          ref: "${{ github.event.pull_request.merge_commit_sha }}"
+          persist-credentials: false
+          ref: "${{ github.event.pull_request.head.sha }}"
 
       - name: Check all submodules are ancestors of origin/HEAD or configured branch
         run: |

.gitmodules

@@ -1,7 +1,3 @@
-[submodule "go-ethereum"]
-	path = go-ethereum
-	url = https://github.com/EspressoSystems/geth-arbitrum.git
-	branch = integration
 [submodule "fastcache"]
 	path = fastcache
 	url = https://github.com/OffchainLabs/fastcache.git
@@ -25,6 +21,9 @@
 	path = nitro-testnode
 	url = https://github.com/EspressoSystems/nitro-testnode
 	branch = celestia-integration
+[submodule "bold"]
+	path = bold
+	url = https://github.com/OffchainLabs/bold.git
 [submodule "arbitrator/langs/rust"]
 	path = arbitrator/langs/rust
 	url = https://github.com/OffchainLabs/stylus-sdk-rs.git
@@ -37,3 +36,7 @@
 [submodule "safe-smart-account"]
 	path = safe-smart-account
 	url = https://github.com/safe-global/safe-smart-account.git
+[submodule "go-ethereum"]
+	path = go-ethereum
+	url = https://github.com/OffchainLabs/go-ethereum.git
+	branch = nitro_v3.3.1

.golangci.yml

@@ -16,6 +16,7 @@ linters:
   enable:
     - asciicheck  # check for non-ascii characters
     - errorlint   # enure error wrapping is safely done
+    - gci         # keep imports sorted deterministically
     - gocritic    # check for certain simplifications
     - gofmt       # ensure code is formatted
     - gosec       # check for security concerns
@@ -30,6 +31,13 @@ linters-settings:
     #
     check-type-assertions: true
 
+  gci:
+    sections:
+      - standard
+      - default
+      - prefix(github.com/ethereum/go-ethereum)
+      - prefix(github.com/offchainlabs)
+
   gocritic:
     disabled-tags:
       - experimental

Dockerfile

@@ -45,8 +45,8 @@ FROM wasm-base AS wasm-libs-builder
 # clang / lld used by soft-float wasm
 RUN apt-get update && \
     apt-get install -y clang=1:14.0-55.7~deb12u1 lld=1:14.0-55.7~deb12u1 wabt
-    # pinned rust 1.81.0
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.81.0 --target x86_64-unknown-linux-gnu wasm32-unknown-unknown wasm32-wasi
+# pinned rust 1.80.1
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.80.1 --target x86_64-unknown-linux-gnu wasm32-unknown-unknown wasm32-wasi
 COPY ./Makefile ./
 COPY arbitrator/Cargo.* arbitrator/
 COPY arbitrator/arbutil arbitrator/arbutil
@@ -66,7 +66,7 @@ COPY --from=wasm-libs-builder /workspace/ /
 
 FROM wasm-base AS wasm-bin-builder
 # pinned go version
-RUN curl -L https://golang.org/dl/go1.21.10.linux-`dpkg --print-architecture`.tar.gz | tar -C /usr/local -xzf -
+RUN curl -L https://golang.org/dl/go1.23.1.linux-`dpkg --print-architecture`.tar.gz | tar -C /usr/local -xzf -
 COPY ./Makefile ./go.mod ./go.sum ./
 COPY ./arbcompress ./arbcompress
 COPY ./arbos ./arbos
@@ -204,6 +204,8 @@ RUN apt-get update && apt-get install -y unzip wget curl
 WORKDIR /workspace/machines
 # Download WAVM machines
 COPY ./scripts/download-machine.sh .
+COPY ./scripts/download-machine-celestia.sh .
+RUN chmod +x ./download-machine.sh ./download-machine-celestia.sh
 #RUN ./download-machine.sh consensus-v1-rc1 0xbb9d58e9527566138b682f3a207c0976d5359837f6e330f4017434cca983ff41
 #RUN ./download-machine.sh consensus-v2.1 0x9d68e40c47e3b87a8a7e6368cc52915720a6484bb2f47ceabad7e573e3a11232
 #RUN ./download-machine.sh consensus-v3 0x53c288a0ca7100c0f2db8ab19508763a51c7fd1be125d376d940a65378acaee7
@@ -225,8 +227,10 @@ COPY ./scripts/download-machine.sh .
 RUN ./download-machine.sh consensus-v30 0xb0de9cb89e4d944ae6023a3b62276e54804c242fd8c4c2d8e6cc4450f5fa8b1b && true
 RUN ./download-machine.sh consensus-v31 0x260f5fa5c3176a856893642e149cf128b5a8de9f828afec8d11184415dd8dc69
 RUN ./download-machine.sh consensus-v32 0x184884e1eb9fefdc158f6c8ac912bb183bf3cf83f0090317e0bc4ac5860baa39
+RUN ./download-machine-celestia.sh v3.2.1-rc.1 0xe81f986823a85105c5fd91bb53b4493d38c0c26652d23f76a7405ac889908287 celestiaorg
+RUN ./download-machine-celestia.sh v3.3.2 0xaf1dbdfceb871c00bfbb1675983133df04f0ed04e89647812513c091e3a982b3 celestiaorg
 
-FROM golang:1.21.10-bookworm AS node-builder
+FROM golang:1.23.1-bookworm AS node-builder
 WORKDIR /workspace
 ARG version=""
 ARG datetime=""
@@ -240,6 +244,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
 COPY go.mod go.sum ./
 COPY go-ethereum/go.mod go-ethereum/go.sum go-ethereum/
 COPY fastcache/go.mod fastcache/go.sum fastcache/
+COPY bold/go.mod bold/go.sum bold/
 RUN go mod download
 COPY . ./
 COPY --from=contracts-builder workspace/contracts/build/ contracts/build/

LICENSE.md

@@ -19,10 +19,10 @@ Additional Use Grant: You may use the Licensed Work in a production environment
                       validating the correctness of the posted chain state, or to deploy
                       and operate (x) a blockchain that settles to a Covered Arbitrum Chain
                       or (y) a blockchain in accordance with, and subject to, the [Arbitrum
-                      Expansion Program Term of Use](https://docs.arbitrum.foundation/assets/files/Arbitrum%20Expansion%20Program%20Jan182024-4f08b0c2cb476a55dc153380fa3e64b0.pdf). For purposes of this
+                      Expansion Program Term of Use](https://docs.arbitrum.foundation/aep/ArbitrumExpansionProgramTerms.pdf). For purposes of this
                       Additional Use Grant, the "Covered Arbitrum Chains" are 
                       (a) Arbitrum One (chainid:42161), Arbitrum Nova (chainid:42170), 
-                      rbitrum Rinkeby testnet/Rinkarby (chainid:421611),Arbitrum Nitro
+                      Arbitrum Rinkeby testnet/Rinkarby (chainid:421611),Arbitrum Nitro
                       Goerli testnet (chainid:421613), and Arbitrum Sepolia Testnet
                       (chainid:421614); (b) any future blockchains authorized to be
                       designated as Covered Arbitrum Chains by the decentralized autonomous

Makefile

@@ -149,13 +149,16 @@ stylus_test_erc20_wasm            = $(call get_stylus_test_wasm,erc20)
 stylus_test_erc20_src             = $(call get_stylus_test_rust,erc20)
 stylus_test_read-return-data_wasm = $(call get_stylus_test_wasm,read-return-data)
 stylus_test_read-return-data_src  = $(call get_stylus_test_rust,read-return-data)
+stylus_test_hostio-test_wasm      = $(call get_stylus_test_wasm,hostio-test)
+stylus_test_hostio-test_src       = $(call get_stylus_test_rust,hostio-test)
 
-stylus_test_wasms = $(stylus_test_keccak_wasm) $(stylus_test_keccak-100_wasm) $(stylus_test_fallible_wasm) $(stylus_test_storage_wasm) $(stylus_test_multicall_wasm) $(stylus_test_log_wasm) $(stylus_test_create_wasm) $(stylus_test_math_wasm) $(stylus_test_sdk-storage_wasm) $(stylus_test_erc20_wasm) $(stylus_test_read-return-data_wasm) $(stylus_test_evm-data_wasm) $(stylus_test_bfs:.b=.wasm)
+stylus_test_wasms = $(stylus_test_keccak_wasm) $(stylus_test_keccak-100_wasm) $(stylus_test_fallible_wasm) $(stylus_test_storage_wasm) $(stylus_test_multicall_wasm) $(stylus_test_log_wasm) $(stylus_test_create_wasm) $(stylus_test_math_wasm) $(stylus_test_sdk-storage_wasm) $(stylus_test_erc20_wasm) $(stylus_test_read-return-data_wasm) $(stylus_test_evm-data_wasm) $(stylus_test_hostio-test_wasm) $(stylus_test_bfs:.b=.wasm)
 stylus_benchmarks = $(wildcard $(stylus_dir)/*.toml $(stylus_dir)/src/*.rs) $(stylus_test_wasms)
 
 espresso_crypto_dir = ./espressocrypto/lib/espresso-crypto-helper
 espresso_crypto_files = $(wildcard $(espresso_crypto_dir)/*.toml $(espresso_crypto_dir)/src/*.rs)
 espresso_crypto_lib = $(output_root)/lib/libespresso_crypto_helper.a
+CBROTLI_WASM_BUILD_ARGS ?=-d
 
 # user targets
 .PHONY: build-espresso-crypto-lib
@@ -294,6 +297,7 @@ clean:
 	rm -f arbitrator/wasm-libraries/soft-float/SoftFloat/build/Wasm-Clang/*.a
 	rm -f arbitrator/wasm-libraries/forward/*.wat
 	rm -rf arbitrator/stylus/tests/*/target/ arbitrator/stylus/tests/*/*.wasm
+	rm -rf brotli/buildfiles
 	@rm -rf contracts/build contracts/cache solgen/go/
 	@rm -f .make/*
 	rm -rf brotli/buildfiles
@@ -498,6 +502,10 @@ $(stylus_test_erc20_wasm): $(stylus_test_erc20_src)
 	$(cargo_nightly) --manifest-path $< --release --config $(stylus_cargo)
 	@touch -c $@ # cargo might decide to not rebuild the binary
 
+$(stylus_test_hostio-test_wasm): $(stylus_test_hostio-test_src)
+	$(cargo_nightly) --manifest-path $< --release --config $(stylus_cargo)
+	@touch -c $@ # cargo might decide to not rebuild the binary
+
 contracts/test/prover/proofs/float%.json: $(arbitrator_cases)/float%.wasm $(prover_bin) $(output_latest)/soft-float.wasm
 	$(prover_bin) $< -l $(output_latest)/soft-float.wasm -o $@ -b --allow-hostapi --require-success
 
@@ -588,9 +596,9 @@ contracts/test/prover/proofs/%.json: $(arbitrator_cases)/%.wasm $(prover_bin)
 	@touch $@
 
 .make/cbrotli-wasm: $(DEP_PREDICATE) $(ORDER_ONLY_PREDICATE) .make
-	test -f target/lib-wasm/libbrotlicommon-static.a || ./scripts/build-brotli.sh -w -d
-	test -f target/lib-wasm/libbrotlienc-static.a || ./scripts/build-brotli.sh -w -d
-	test -f target/lib-wasm/libbrotlidec-static.a || ./scripts/build-brotli.sh -w -d
+	test -f target/lib-wasm/libbrotlicommon-static.a || ./scripts/build-brotli.sh -w $(CBROTLI_WASM_BUILD_ARGS)
+	test -f target/lib-wasm/libbrotlienc-static.a || ./scripts/build-brotli.sh -w $(CBROTLI_WASM_BUILD_ARGS)
+	test -f target/lib-wasm/libbrotlidec-static.a || ./scripts/build-brotli.sh -w $(CBROTLI_WASM_BUILD_ARGS)
 	@touch $@
 
 .make/wasm-lib: $(DEP_PREDICATE) arbitrator/wasm-libraries/soft-float/SoftFloat/build/Wasm-Clang/softfloat.a  $(ORDER_ONLY_PREDICATE) .make

README.md

@@ -47,7 +47,7 @@ Nitro is currently licensed under a [Business Source License](./LICENSE.md), sim
 
 The Additional Use Grant also permits the deployment of the Nitro software, in a permissionless fashion and without cost, as a new blockchain provided that the chain settles to either Arbitrum One or Arbitrum Nova.
 
-For those that prefer to deploy the Nitro software either directly on Ethereum (i.e. an L2) or have it settle to another Layer-2 on top of Ethereum, the [Arbitrum Expansion Program (the "AEP")](https://docs.arbitrum.foundation/assets/files/Arbitrum%20Expansion%20Program%20Jan182024-4f08b0c2cb476a55dc153380fa3e64b0.pdf) was recently established. The AEP allows for the permissionless deployment in the aforementioned fashion provided that 10% of net revenue (as more fully described in the AEP) is contributed back to the Arbitrum community in accordance with the requirements of the AEP.
+For those that prefer to deploy the Nitro software either directly on Ethereum (i.e. an L2) or have it settle to another Layer-2 on top of Ethereum, the [Arbitrum Expansion Program (the "AEP")](https://docs.arbitrum.foundation/aep/ArbitrumExpansionProgramTerms.pdf) was recently established. The AEP allows for the permissionless deployment in the aforementioned fashion provided that 10% of net revenue (as more fully described in the AEP) is contributed back to the Arbitrum community in accordance with the requirements of the AEP.
 
 ## Contact
 

arbcompress/native.go

@@ -7,11 +7,12 @@
 package arbcompress
 
 /*
-#cgo CFLAGS: -g -Wall -I${SRCDIR}/../target/include/
+#cgo CFLAGS: -g -I${SRCDIR}/../target/include/
 #cgo LDFLAGS: ${SRCDIR}/../target/lib/libstylus.a -lm
 #include "arbitrator.h"
 */
 import "C"
+
 import (
 	"errors"
 	"fmt"