Index data #13206
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Index data | |
| on: | |
| schedule: | |
| - cron: "*/30 * * * *" | |
| workflow_dispatch: | |
| jobs: | |
| index: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: webfactory/ssh-agent@v0.5.4 | |
| with: | |
| ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| - name: Checkout neume-network/data | |
| uses: actions/checkout@v3 | |
| with: | |
| path: neume-network-data | |
| ssh-key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| - name: "Generate a block logs crawl path" | |
| run: | | |
| cd ./neume-network-data/scripts && \ | |
| chmod +x * && \ | |
| ./generateBlockLogsCrawlPath > ../steps/0.mjs && \ | |
| cd ../.. | |
| env: | |
| DATA_DIR: data | |
| - name: Checkout neume-network/core | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: neume-network/core | |
| ref: ${{ github.ref_name }} | |
| path: neume-network-core | |
| ssh-key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| - name: Use Node.js 14.x | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 14.x | |
| cache: "npm" | |
| cache-dependency-path: neume-network-core/package-lock.json | |
| - name: Install neume-network/core dependencies | |
| working-directory: ./neume-network-core | |
| run: npm ci | |
| shell: bash | |
| - name: Run steps | |
| run: | | |
| neume-network-core/neume.mjs \ | |
| --path neume-network-data/steps/0.mjs \ | |
| --config neume-network-data/config.mjs && \ | |
| neume-network-core/neume.mjs \ | |
| --path neume-network-data/steps/1.mjs \ | |
| --config neume-network-data/config.mjs && \ | |
| neume-network-core/neume.mjs \ | |
| --path neume-network-data/steps/2.mjs \ | |
| --config neume-network-data/config.mjs | |
| shell: bash | |
| env: | |
| DATA_DIR: data | |
| RPC_HTTP_HOST: ${{ secrets.RPC_HTTP_HOST }} | |
| RPC_API_KEY: "" | |
| IPFS_HTTPS_GATEWAY: ${{ secrets.IPFS_HTTPS_GATEWAY }} | |
| IPFS_HTTPS_GATEWAY_KEY: ${{ secrets.IPFS_HTTPS_GATEWAY_KEY }} | |
| EXTRACTION_WORKER_CONCURRENCY: 80 | |
| DEBUG: neume-network-* | |
| ARWEAVE_HTTPS_GATEWAY: https://arweave.net | |
| - name: Update neume-network/data files | |
| shell: bash | |
| run: | | |
| test -f data/call-block-logs-extraction && rm data/call-block-logs-extraction || \ | |
| test -f data/call-block-logs-transformation && cat data/call-block-logs-transformation >> neume-network-data/results/call-block-logs-transformation || \ | |
| test -f data/soundxyz-filter-contracts-transformation && cat data/soundxyz-filter-contracts-transformation >> neume-network-data/results/soundxyz-filter-contracts-transformation || \ | |
| test -f data/soundxyz-call-tokenuri-extraction && cat data/soundxyz-call-tokenuri-extraction >> neume-network-data/results/soundxyz-call-tokenuri-extraction || \ | |
| test -f data/soundxyz-call-tokenuri-transformation && cat data/soundxyz-call-tokenuri-transformation >> neume-network-data/results/soundxyz-call-tokenuri-transformation || \ | |
| test -f data/sound-protocol-call-tokenuri-extraction && cat data/sound-protocol-call-tokenuri-extraction >> neume-network-data/results/sound-protocol-call-tokenuri-extraction || \ | |
| test -f data/sound-protocol-call-tokenuri-transformation && cat data/sound-protocol-call-tokenuri-transformation >> neume-network-data/results/sound-protocol-call-tokenuri-transformation || \ | |
| test -f data/soundxyz-get-tokenuri-extraction && cat data/soundxyz-get-tokenuri-extraction >> neume-network-data/results/soundxyz-get-tokenuri-extraction || \ | |
| test -f data/soundxyz-get-tokenuri-transformation && cat data/soundxyz-get-tokenuri-transformation >> neume-network-data/results/soundxyz-get-tokenuri-transformation || \ | |
| test -f data/sound-protocol-get-tokenuri-extraction && cat data/sound-protocol-get-tokenuri-extraction >> neume-network-data/results/sound-protocol-get-tokenuri-extraction || \ | |
| test -f data/sound-protocol-get-tokenuri-transformation && cat data/sound-protocol-get-tokenuri-transformation >> neume-network-data/results/sound-protocol-get-tokenuri-transformation || \ | |
| test -f data/soundxyz-metadata-extraction && cat data/soundxyz-metadata-extraction >> neume-network-data/results/soundxyz-metadata-extraction || \ | |
| test -f data/soundxyz-metadata-transformation && cat data/soundxyz-metadata-transformation >> neume-network-data/results/soundxyz-metadata-transformation || \ | |
| test -f data/catalog-call-tokenuri-extraction && cat data/catalog-call-tokenuri-extraction >> neume-network-data/results/catalog-call-tokenuri-extraction || \ | |
| test -f data/catalog-get-tokenuri-extraction && cat data/catalog-get-tokenuri-extraction >> neume-network-data/results/catalog-get-tokenuri-extraction || \ | |
| test -f data/noizd-call-tokenuri-extraction && cat data/noizd-call-tokenuri-extraction >> neume-network-data/results/noizd-call-tokenuri-extraction || \ | |
| test -f data/noizd-get-tokenuri-extraction && cat data/noizd-get-tokenuri-extraction >> neume-network-data/results/noizd-get-tokenuri-extraction || \ | |
| test -f data/mintsongs-call-tokenuri-extraction && cat data/mintsongs-call-tokenuri-extraction >> neume-network-data/results/mintsongs-call-tokenuri-extraction || \ | |
| test -f data/mintsongs-get-tokenuri-extraction && cat data/mintsongs-get-tokenuri-extraction >> neume-network-data/results/mintsongs-get-tokenuri-extraction || \ | |
| test -f data/zora-call-tokenmetadatauri-extraction && cat data/zora-call-tokenmetadatauri-extraction >> neume-network-data/results/zora-call-tokenmetadatauri-extraction || \ | |
| test -f data/zora-call-tokenmetadatauri-transformation && cat data/zora-call-tokenmetadatauri-transformation >> neume-network-data/results/zora-call-tokenmetadatauri-transformation || \ | |
| test -f data/zora-call-tokenuri-extraction && cat data/zora-call-tokenuri-extraction >> neume-network-data/results/zora-call-tokenuri-extraction || \ | |
| test -f data/zora-get-tokenuri-extraction && cat data/zora-get-tokenuri-extraction >> neume-network-data/results/zora-get-tokenuri-extraction || \ | |
| test -f data/zora-get-tokenuri-transformation && cat data/zora-get-tokenuri-transformation >> neume-network-data/results/zora-get-tokenuri-transformation || true | |
| - uses: sergeysova/jq-action@v2 | |
| with: | |
| cmd: jq -s 'add' neume-network-data/results/music-os-accumulator-extraction data/music-os-accumulator-extraction > neume-network-data/results/music-os-accumulator-extraction-new | |
| - name: Rename new music-os-accumulator-file | |
| shell: bash | |
| run: mv neume-network-data/results/music-os-accumulator-extraction-new neume-network-data/results/music-os-accumulator-extraction | |
| - name: Test lastBlockNumber file | |
| run: test 0 -eq $(grep -c 'echo ""' neume-network-data/scripts/lastBlockNumber) | |
| - name: Update neume-network/data | |
| run: | | |
| cd neume-network-data | |
| git config user.email info@neume.network | |
| git config user.name neume-network | |
| git add results/* scripts/lastBlockNumber | |
| git commit -m "Update from ${{github.server_url}}/${{github.repository}}/commit/${{github.sha}}" -m "${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}" | |
| git push origin HEAD:${{ github.ref_name }} | |