wip: setup environment for running dips locally

.env

@@ -21,14 +21,17 @@
 #   rewards-eligibility   REO eligibility oracle node
 #   indexing-payments     dipper + iisa (requires GHCR auth — see README)
 # Default: profiles that work out of the box.
-COMPOSE_PROFILES=rewards-eligibility,block-oracle,explorer
+COMPOSE_PROFILES=block-oracle
 # All profiles (indexing-payments requires GHCR auth — see README):
 #COMPOSE_PROFILES=rewards-eligibility,block-oracle,explorer,indexing-payments
 
 # --- Dev overrides ---
 # Uncomment and extend to build services from local source.
 # See compose/dev/README.md for available overrides.
-#COMPOSE_FILE=docker-compose.yaml:compose/dev/graph-node.yaml
+COMPOSE_FILE=docker-compose.yaml:compose/dev/dips.yaml
+INDEXER_SERVICE_SOURCE_ROOT=../indexer-rs
+INDEXER_AGENT_SOURCE_ROOT=../indexer
+DIPPER_SOURCE_ROOT=../dipper
 
 # indexer components versions
 GRAPH_NODE_VERSION=v0.37.0
@@ -51,7 +54,7 @@ ELIGIBILITY_ORACLE_COMMIT=84710857394d3419f83dcbf6687a91f415cc1625
 
 # network components versions
 BLOCK_ORACLE_COMMIT=3a3a425ff96130c3842cee7e43d06bbe3d729aed
-CONTRACTS_COMMIT=0003fe3adf7394df5c9ac1cf4ef600c96a61259f
+CONTRACTS_COMMIT=63094533b29b3a0628824d6e6eaa0f05e14f57b6
 NETWORK_SUBGRAPH_COMMIT=5b6c22089a2e55db16586a19cbf6e1d73a93c7b9
 TAP_CONTRACTS_COMMIT=e3351e70b3e5d9821bc0aaa90bb2173ca2a77af7
 TAP_SUBGRAPH_COMMIT=cf7279f60433bf9a9d897ec2548c13c0607234cc
@@ -99,6 +102,7 @@ BLOCK_EXPLORER=${BLOCK_EXPLORER_PORT}
 # Indexing Payments (used with indexing-payments override)
 DIPPER_ADMIN_RPC_PORT=9000
 DIPPER_INDEXER_RPC_PORT=9001
+INDEXER_SERVICE_DIPS_RPC_PORT=7602
 
 ## Chain config
 CHAIN_ID=1337
@@ -124,7 +128,7 @@ SUBGRAPH_2="9p1TRzaccKzWBN4P6YEwEUxYwJn6HwPxf5dKXK2NYxgS"
 
 # REO (Rewards Eligibility Oracle)
 # Set to 1 to deploy and configure the REO contract (Phase 4). Unset or 0 to skip.
-REO_ENABLED=1
+REO_ENABLED=0
 # eligibilityPeriod: how long an indexer stays eligible after renewal (seconds)
 REO_ELIGIBILITY_PERIOD=300
 # oracleUpdateTimeout: fail-safe — if no oracle update for this long, all indexers eligible (seconds)

compose/dev/README.md

@@ -31,5 +31,6 @@ Then `docker compose up -d` applies the overrides automatically.
 | `eligibility-oracle.yaml` | eligibility-oracle-node          | `REO_BINARY`                                           |
 | `dipper.yaml`             | dipper                           | `DIPPER_BINARY`                                        |
 | `iisa.yaml`               | iisa                             | `IISA_VERSION=local`                                   |
+| `dips.yaml`               | indexer-service, indexer-agent    | `INDEXER_SERVICE_SOURCE_ROOT`, `INDEXER_AGENT_SOURCE_ROOT` |
 
 See each file's header comments for details.

compose/dev/dips.yaml

@@ -0,0 +1,78 @@
+# DIPs Development Override
+#
+# Overrides indexer-service and indexer-agent for DIPs development:
+#   - indexer-service: built from local source with [dips] config section
+#   - indexer-agent: hot-reload from local source with DIPs env vars
+#
+# Prerequisites:
+#   - Sibling checkouts: ../indexer-rs, ../indexer, ../dipper
+#   - indexing-payments profile enabled (for dipper service)
+#
+# Activate via .env:
+#   COMPOSE_PROFILES=indexing-payments,block-oracle
+#   COMPOSE_FILE=docker-compose.yaml:compose/dev/dips.yaml
+#   INDEXER_SERVICE_SOURCE_ROOT=../indexer-rs
+#   INDEXER_AGENT_SOURCE_ROOT=../indexer
+#   DIPPER_SOURCE_ROOT=../dipper
+
+services:
+  indexer-service:
+    build:
+      target: "wrapper"
+      dockerfile_inline: |
+        FROM rust:1-slim-bookworm AS wrapper
+        RUN apt-get update \
+            && apt-get install -y --no-install-recommends \
+                build-essential curl git jq pkg-config \
+                protobuf-compiler libssl-dev libsasl2-dev \
+            && rm -rf /var/lib/apt/lists/*
+    entrypoint: ["bash", "/opt/run-dips.sh"]
+    volumes:
+      - ${INDEXER_SERVICE_SOURCE_ROOT:?Set INDEXER_SERVICE_SOURCE_ROOT to local indexer-rs checkout}:/opt/source
+      - ./containers/indexer/indexer-service/dev/run-dips.sh:/opt/run-dips.sh:ro
+      - ./containers/shared:/opt/shared:ro
+      - ./.env:/opt/config/.env:ro
+      - config-local:/opt/config:ro
+    ports:
+      - "${INDEXER_SERVICE_PORT}:7601"
+      - "${INDEXER_SERVICE_DIPS_RPC_PORT}:7602"
+    environment:
+      RUST_LOG: info,indexer_service_rs=info,indexer_monitor=debug,indexer_dips=debug
+      RUST_BACKTRACE: 1
+      SQLX_OFFLINE: "true"
+    healthcheck:
+      interval: 10s
+      retries: 600
+      test: curl -f http://127.0.0.1:7601/
+
+  indexer-agent:
+    entrypoint: ["bash", "-cl", "/opt/run-dips.sh"]
+    ports:
+      - "${INDEXER_MANAGEMENT_PORT}:7600"
+      - 9230:9230
+    volumes:
+      - ./containers/indexer/indexer-agent/dev/run-dips.sh:/opt/run-dips.sh:ro
+      - ${INDEXER_AGENT_SOURCE_ROOT:?Set INDEXER_AGENT_SOURCE_ROOT to local indexer checkout}:/opt/indexer-agent-source-root
+
+  dipper:
+    profiles: []
+    build:
+      dockerfile_inline: |
+        FROM rust:1-slim-bookworm
+        RUN apt-get update \
+            && apt-get install -y --no-install-recommends \
+                build-essential ca-certificates clang cmake curl git jq lld \
+                pkg-config libssl-dev protobuf-compiler \
+            && rm -rf /var/lib/apt/lists/*
+        ENV CC=clang CXX=clang++ RUSTFLAGS="-C link-arg=-fuse-ld=lld"
+    entrypoint: ["bash", "/opt/run.sh"]
+    depends_on:
+      block-oracle: { condition: service_healthy }
+      postgres: { condition: service_healthy }
+      gateway: { condition: service_healthy }
+    volumes:
+      - ${DIPPER_SOURCE_ROOT:?Set DIPPER_SOURCE_ROOT to local dipper checkout}:/opt/source
+      - ./containers/indexing-payments/dipper/run.sh:/opt/run.sh:ro
+      - ./containers/shared:/opt/shared:ro
+      - ./.env:/opt/config/.env:ro
+      - config-local:/opt/config:ro

containers/indexer/indexer-agent/dev/run-dips.sh

@@ -0,0 +1,89 @@
+#!/bin/bash
+set -xeu
+. /opt/config/.env
+
+. /opt/shared/lib.sh
+
+token_address=$(contract_addr L2GraphToken.address horizon)
+staking_address=$(contract_addr HorizonStaking.address horizon)
+indexer_staked="$(cast call "--rpc-url=http://chain:${CHAIN_RPC_PORT}" \
+  "${staking_address}" 'hasStake(address) (bool)' "${RECEIVER_ADDRESS}")"
+echo "indexer_staked=${indexer_staked}"
+if [ "${indexer_staked}" = "false" ]; then
+  cast send "--rpc-url=http://chain:${CHAIN_RPC_PORT}" --confirmations=0 "--mnemonic=${MNEMONIC}" \
+    --value=1ether "${RECEIVER_ADDRESS}"
+  cast send "--rpc-url=http://chain:${CHAIN_RPC_PORT}" --confirmations=0 "--mnemonic=${MNEMONIC}" \
+    "${token_address}" 'transfer(address,uint256)' "${RECEIVER_ADDRESS}" '100000000000000000000000'
+  cast send "--rpc-url=http://chain:${CHAIN_RPC_PORT}" --confirmations=0 "--private-key=${RECEIVER_SECRET}" \
+    "${token_address}" 'approve(address,uint256)' "${staking_address}" '100000000000000000000000'
+  cast send "--rpc-url=http://chain:${CHAIN_RPC_PORT}" --confirmations=0 "--private-key=${RECEIVER_SECRET}" \
+    "${staking_address}" 'stake(uint256)' '100000000000000000000000'
+fi
+
+export INDEXER_AGENT_HORIZON_ADDRESS_BOOK=/opt/config/horizon.json
+export INDEXER_AGENT_SUBGRAPH_SERVICE_ADDRESS_BOOK=/opt/config/subgraph-service.json
+export INDEXER_AGENT_TAP_ADDRESS_BOOK=/opt/config/tap-contracts.json
+export INDEXER_AGENT_EPOCH_SUBGRAPH_ENDPOINT="http://graph-node:${GRAPH_NODE_GRAPHQL_PORT}/subgraphs/name/block-oracle"
+export INDEXER_AGENT_GATEWAY_ENDPOINT="http://gateway:${GATEWAY_PORT}"
+export INDEXER_AGENT_GRAPH_NODE_QUERY_ENDPOINT="http://graph-node:${GRAPH_NODE_GRAPHQL_PORT}"
+export INDEXER_AGENT_GRAPH_NODE_ADMIN_ENDPOINT="http://graph-node:${GRAPH_NODE_ADMIN_PORT}"
+export INDEXER_AGENT_GRAPH_NODE_STATUS_ENDPOINT="http://graph-node:${GRAPH_NODE_STATUS_PORT}/graphql"
+export INDEXER_AGENT_IPFS_ENDPOINT="http://ipfs:${IPFS_RPC_PORT}"
+export INDEXER_AGENT_INDEXER_ADDRESS="${RECEIVER_ADDRESS}"
+export INDEXER_AGENT_INDEXER_MANAGEMENT_PORT="${INDEXER_MANAGEMENT_PORT}"
+export INDEXER_AGENT_INDEX_NODE_IDS=default
+export INDEXER_AGENT_INDEXER_GEO_COORDINATES="1 1"
+export INDEXER_AGENT_VOUCHER_REDEMPTION_THRESHOLD=0.01
+export INDEXER_AGENT_NETWORK_SUBGRAPH_ENDPOINT="http://graph-node:${GRAPH_NODE_GRAPHQL_PORT}/subgraphs/name/graph-network"
+export INDEXER_AGENT_NETWORK_PROVIDER="http://chain:${CHAIN_RPC_PORT}"
+export INDEXER_AGENT_MNEMONIC="${INDEXER_MNEMONIC}"
+export INDEXER_AGENT_POSTGRES_DATABASE=indexer_components_1
+export INDEXER_AGENT_POSTGRES_HOST=postgres
+export INDEXER_AGENT_POSTGRES_PORT="${POSTGRES_PORT}"
+export INDEXER_AGENT_POSTGRES_USERNAME=postgres
+export INDEXER_AGENT_POSTGRES_PASSWORD=
+export INDEXER_AGENT_PUBLIC_INDEXER_URL="http://indexer-service:${INDEXER_SERVICE_PORT}"
+export INDEXER_AGENT_TAP_SUBGRAPH_ENDPOINT="http://graph-node:${GRAPH_NODE_GRAPHQL_PORT}/subgraphs/name/semiotic/tap"
+export INDEXER_AGENT_MAX_PROVISION_INITIAL_SIZE=200000
+export INDEXER_AGENT_CONFIRMATION_BLOCKS=1
+export INDEXER_AGENT_LOG_LEVEL=trace
+export INDEXER_AGENT_POLLING_INTERVAL=10000
+
+# DIPs configuration
+export INDEXER_AGENT_ENABLE_DIPS=true
+export INDEXER_AGENT_DIPS_EPOCHS_MARGIN=1
+export INDEXER_AGENT_DIPPER_ENDPOINT="http://dipper:${DIPPER_INDEXER_RPC_PORT}"
+export INDEXER_AGENT_DIPS_ALLOCATION_AMOUNT=1
+
+cd /opt/indexer-agent-source-root
+mkdir -p ./config/
+cat >./config/config.yaml <<-EOF
+networkIdentifier: "hardhat"
+indexerOptions:
+  geoCoordinates: [48.4682, -123.524]
+  defaultAllocationAmount: 10000
+  allocationManagementMode: "auto"
+  restakeRewards: true
+  poiDisputeMonitoring: false
+  voucherRedemptionThreshold: 0.00001
+  voucherRedemptionBatchThreshold: 10
+  rebateClaimThreshold: 0.00001
+  rebateClaimBatchThreshold: 10
+subgraphs:
+  maxBlockDistance: 5000
+  freshnessSleepMilliseconds: 1000
+enableDips: true
+dipperEndpoint: "http://dipper:${DIPPER_INDEXER_RPC_PORT}"
+dipsAllocationAmount: 1
+dipsEpochsMargin: 1
+EOF
+cat config/config.yaml
+
+nodemon --watch . \
+--ext ts \
+--legacy-watch \
+--delay 4 \
+--verbose \
+--exec "
+NODE_OPTIONS=\"--inspect=0.0.0.0:9230\"
+tsx packages/indexer-agent/src/index.ts start"

containers/indexer/indexer-service/dev/run-dips.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+set -eu
+
+. /opt/config/.env
+. /opt/shared/lib.sh
+
+tap_verifier=$(contract_addr TAPVerifier tap-contracts)
+graph_tally_verifier=$(contract_addr GraphTallyCollector.address horizon)
+subgraph_service=$(contract_addr SubgraphService.address subgraph-service)
+recurring_collector=$(contract_addr RecurringCollector.address horizon)
+
+cat >/opt/config.toml <<-EOF
+[indexer]
+indexer_address = "${RECEIVER_ADDRESS}"
+operator_mnemonic = "${INDEXER_MNEMONIC}"
+
+[database]
+postgres_url = "postgresql://postgres@postgres:${POSTGRES_PORT}/indexer_components_1"
+
+[graph_node]
+query_url = "http://graph-node:${GRAPH_NODE_GRAPHQL_PORT}"
+status_url = "http://graph-node:${GRAPH_NODE_STATUS_PORT}/graphql"
+
+[subgraphs.network]
+query_url = "http://graph-node:${GRAPH_NODE_GRAPHQL_PORT}/subgraphs/name/graph-network"
+recently_closed_allocation_buffer_secs = 60
+syncing_interval_secs = 30
+
+[subgraphs.escrow]
+query_url = "http://graph-node:${GRAPH_NODE_GRAPHQL_PORT}/subgraphs/name/semiotic/tap"
+syncing_interval_secs = 30
+
+[blockchain]
+chain_id = 1337
+receipts_verifier_address = "${tap_verifier}"
+receipts_verifier_address_v2 = "${graph_tally_verifier}"
+subgraph_service_address = "${subgraph_service}"
+
+[service]
+free_query_auth_token = "freestuff"
+host_and_port = "0.0.0.0:${INDEXER_SERVICE_PORT}"
+url_prefix = "/"
+serve_network_subgraph = false
+serve_escrow_subgraph = false
+ipfs_url = "http://ipfs:${IPFS_RPC_PORT}"
+
+[tap]
+max_amount_willing_to_lose_grt = 1
+
+[tap.rav_request]
+timestamp_buffer_secs = 15
+
+[tap.sender_aggregator_endpoints]
+${ACCOUNT0_ADDRESS} = "http://tap-aggregator:${TAP_AGGREGATOR_PORT}"
+
+[horizon]
+enabled = true
+
+[dips]
+host = "0.0.0.0"
+port = "${INDEXER_SERVICE_DIPS_RPC_PORT}"
+recurring_collector = "${recurring_collector}"
+allowed_payers = ["${ACCOUNT0_ADDRESS}"]
+
+price_per_entity = "1000"
+
+[dips.price_per_epoch]
+"eip155:1" = "100"
+"eip155:1337" = "100"
+
+[dips.additional_networks]
+"eip155:1337" = "hardhat"
+EOF
+cat /opt/config.toml
+
+cd /opt/source
+cargo build --bin indexer-service-rs
+exec ./target/debug/indexer-service-rs --config=/opt/config.toml

containers/indexing-payments/dipper/run.sh

@@ -79,4 +79,11 @@ echo "=== Generated config.json ===" >&2
 cat config.json >&2
 echo "===========================" >&2
 
-dipper-service ./config.json
+# Build from source if mounted, otherwise use pre-built binary
+if [ -d /opt/source ] && [ -f /opt/source/Cargo.toml ]; then
+  cd /opt/source
+  cargo build --bin dipper-service --release
+  exec ./target/release/dipper-service "$OLDPWD/config.json"
+else
+  exec dipper-service ./config.json
+fi

scripts/add-subgraph.sh

@@ -23,7 +23,7 @@ deployment_hex="$(curl -s -X POST "http://${IPFS_HOST}:${IPFS_RPC_PORT}/api/v0/c
 deployment_hex="${deployment_hex#f01701220}"
 
 echo "deployment_hex=${deployment_hex}"
-gns="$(jq -r '."1337".L2GNS.address' subgraph-service.json)"
+gns="$(docker exec graph-node cat /opt/config/subgraph-service.json | jq -r '."1337".L2GNS.address')"
 
 # https://github.com/graphprotocol/contracts/blob/3eb16c80d4652c238d3e6b2c396da712af5072b4/packages/sdk/src/deployments/network/actions/gns.ts#L38
 cast send --rpc-url="http://${CHAIN_HOST}:${CHAIN_RPC_PORT}" --confirmations=0 --mnemonic="${MNEMONIC}" \