Issue 779 (#2141)

* feat: add tx_index to ckb_transactions

Signed-off-by: Miles Zhang <mingchang555@hotmail.com>

* feat: use task to fill tx_index

Signed-off-by: Miles Zhang <mingchang555@hotmail.com>

* feat: save tx_index when sync node

Signed-off-by: Miles Zhang <mingchang555@hotmail.com>

---------

Signed-off-by: Miles Zhang <mingchang555@hotmail.com>

app/models/ckb_sync/new_node_data_processor.rb

@@ -1246,11 +1246,10 @@ def cell_input_attributes(input, ckb_transaction_id, local_block_id,
     def build_ckb_transactions!(node_block, local_block, inputs, outputs, outputs_data)
       cycles = CkbSync::Api.instance.get_block_cycles node_block.header.hash
       ckb_transactions_attributes = []
-      tx_index = 0
       hashes = []
       header_deps = {}
       witnesses = {}
-      node_block.transactions.each do |tx|
+      node_block.transactions.each_with_index do |tx, tx_index|
         attrs = ckb_transaction_attributes(local_block, tx, tx_index)
         if cycles
           attrs[:cycles] = tx_index > 0 ? cycles[tx_index - 1]&.hex : nil
@@ -1263,8 +1262,6 @@ def build_ckb_transactions!(node_block, local_block, inputs, outputs, outputs_da
         inputs[tx_index] = tx.inputs
         outputs[tx_index] = tx.outputs
         outputs_data[tx_index] = tx.outputs_data
-
-        tx_index += 1
       end
       # First update status thus we can use upsert later. otherwise, we may not be able to
       # locate correct record according to tx_hash
@@ -1358,6 +1355,7 @@ def ckb_transaction_attributes(local_block, tx, tx_index)
         is_cellbase: tx_index.zero?,
         live_cell_changes: live_cell_changes(tx, tx_index),
         bytes: tx.serialized_size_in_block,
+        tx_index: tx_index
       }
     end
 

app/models/ckb_transaction.rb

@@ -275,6 +275,7 @@ def log_deletion_chain
 #  bytes             :bigint           default(0)
 #  cycles            :bigint
 #  confirmation_time :integer
+#  tx_index          :integer
 #
 # Indexes
 #

db/migrate/20240822024448_add_tx_index_to_ckb_transaction.rb

@@ -0,0 +1,5 @@
+class AddTxIndexToCkbTransaction < ActiveRecord::Migration[7.0]
+  def change
+    add_column :ckb_transactions, :tx_index, :integer
+  end
+end

db/structure.sql

@@ -9,13 +9,6 @@ SET xmloption = content;
 SET client_min_messages = warning;
 SET row_security = off;
 
---
--- Name: public; Type: SCHEMA; Schema: -; Owner: -
---
-
--- *not* creating schema, since initdb creates it
-
-
 --
 -- Name: btree_gin; Type: EXTENSION; Schema: -; Owner: -
 --
@@ -1350,7 +1343,8 @@ CREATE TABLE public.ckb_transactions (
     tags character varying[] DEFAULT '{}'::character varying[],
     bytes bigint DEFAULT 0,
     cycles bigint,
-    confirmation_time integer
+    confirmation_time integer,
+    tx_index integer
 )
 PARTITION BY LIST (tx_status);
 
@@ -1395,7 +1389,8 @@ CREATE TABLE public.ckb_transactions_committed (
     tags character varying[] DEFAULT '{}'::character varying[],
     bytes bigint DEFAULT 0,
     cycles bigint,
-    confirmation_time integer
+    confirmation_time integer,
+    tx_index integer
 );
 
 
@@ -1420,7 +1415,8 @@ CREATE TABLE public.ckb_transactions_pending (
     tags character varying[] DEFAULT '{}'::character varying[],
     bytes bigint DEFAULT 0,
     cycles bigint,
-    confirmation_time integer
+    confirmation_time integer,
+    tx_index integer
 );
 
 
@@ -1445,7 +1441,8 @@ CREATE TABLE public.ckb_transactions_proposed (
     tags character varying[] DEFAULT '{}'::character varying[],
     bytes bigint DEFAULT 0,
     cycles bigint,
-    confirmation_time integer
+    confirmation_time integer,
+    tx_index integer
 );
 
 
@@ -1470,7 +1467,8 @@ CREATE TABLE public.ckb_transactions_rejected (
     tags character varying[] DEFAULT '{}'::character varying[],
     bytes bigint DEFAULT 0,
     cycles bigint,
-    confirmation_time integer
+    confirmation_time integer,
+    tx_index integer
 );
 
 
@@ -6098,4 +6096,7 @@ INSERT INTO "schema_migrations" (version) VALUES
 ('20240709131020'),
 ('20240709131132'),
 ('20240709131713'),
-('20240709142013');
+('20240709142013'),
+('20240822024448');
+
+

lib/tasks/migration/fill_tx_index.rake

@@ -0,0 +1,43 @@
+namespace :migration do
+  desc "Usage: RAILS_ENV=production bundle exec rake migration:fill_tx_index"
+  task fill_tx_index: :environment do
+    $retry_ids = Set.new
+    @api = CKB::API.new(host: ENV["CKB_NODE_URL"],
+                        timeout_config: {
+                          open_timeout: 1, read_timeout: 3,
+                          write_timeout: 1
+                        })
+    first_tx = CkbTransaction.tx_committed.where(tx_index: nil).order("block_number asc").select(:block_number).first
+    last_tx = CkbTransaction.tx_committed.where(tx_index: nil).order("block_number desc").select(:block_number).first
+
+    (first_tx.block_number..last_tx.block_number).to_a.each_slice(100).to_a.each do |range|
+      fill_missed_tx_index(range, 0)
+    end; nil
+
+    puts "retry IDS:"
+    puts $retry_ids.join(",")
+    puts "done"
+  end
+
+  def fill_missed_tx_index(range, retry_count)
+    request_body =
+      range.map do |number|
+        ["get_block_by_number", number]
+      end
+    response = @api.batch_request(*request_body)
+    attrs = []
+    response.each do |r|
+      r[:transactions].each_with_index do |tx, index|
+        attrs << { tx_hash: tx[:hash], tx_status: "committed", tx_index: index }
+      end
+    end; nil
+    CkbTransaction.upsert_all(attrs, unique_by: %i[tx_status tx_hash])
+  rescue StandardError => _e
+    retry_count += 1
+    if retry_count > 2
+      $retry_ids << range.first
+    else
+      fill_missed_tx_index(range, retry_count)
+    end
+  end
+end