diff --git a/app/models/ckb_sync/new_node_data_processor.rb b/app/models/ckb_sync/new_node_data_processor.rb index b1440ec30..660d46d74 100644 --- a/app/models/ckb_sync/new_node_data_processor.rb +++ b/app/models/ckb_sync/new_node_data_processor.rb @@ -801,11 +801,10 @@ def build_cells_and_locks!( lock_scripts_attributes.map! do |attr| attr.merge!(created_at: Time.current, updated_at: Time.current) end - lock_script_ids = LockScript.insert_all!(lock_scripts_attributes).map do |e| - e["id"] - end + lock_script_ids = LockScript.upsert_all(lock_scripts_attributes, unique_by: :script_hash, returning: [:id]) - lock_script_ids.each do |lock_script_id| + lock_script_ids.each do |row| + lock_script_id = row["id"] lock_script = LockScript.find lock_script_id contract = Contract.find_by code_hash: lock_script.code_hash temp_hash = { script_hash: lock_script&.script_hash, is_contract: false } @@ -815,8 +814,8 @@ def build_cells_and_locks!( contract = Contract.create code_hash: lock_script.script_hash temp_hash = temp_hash.merge contract_id: contract.id end - script = Script.find_or_create_by temp_hash - lock_script.update script_id: script.id + # script = Script.find_or_create_by temp_hash + # lock_script.update script_id: script.id end end @@ -824,11 +823,9 @@ def build_cells_and_locks!( type_scripts_attributes.map! do |attr| attr.merge!(created_at: Time.current, updated_at: Time.current) end - type_script_ids = TypeScript.insert_all!(type_scripts_attributes).map do |e| - e["id"] - end - - type_script_ids.each do |type_script_id| + type_script_ids = TypeScript.upsert_all(type_scripts_attributes, unique_by: :script_hash, returning: [:id]) + type_script_ids.each do |row| + type_script_id = row["id"] type_script = TypeScript.find(type_script_id) temp_hash = { script_hash: type_script&.script_hash, is_contract: false } contract = Contract.find_by code_hash: type_script.code_hash @@ -838,8 +835,8 @@ def build_cells_and_locks!( contract = Contract.create code_hash: type_script.script_hash temp_hash = temp_hash.merge contract_id: contract.id end - script = Script.find_or_create_by temp_hash - type_script.update script_id: script.id + # script = Script.find_or_create_by temp_hash + # type_script.update script_id: script.id end end build_addresses!(outputs, local_block) @@ -1149,7 +1146,7 @@ def cell_output_attributes(output, address, ckb_transaction, local_block, cell_i attrs = { ckb_transaction_id: ckb_transaction["id"], capacity: output.capacity, - occupied_capacity: CkbUtils.cal_cell_min_capacity(lock_script, type_script, output.capacity, binary_data), + occupied_capacity: CkbUtils.cal_cell_min_capacity(output.lock, output.type, binary_data), address_id: address.id, block_id: local_block.id, tx_hash: ckb_transaction["tx_hash"], @@ -1267,7 +1264,7 @@ def build_ckb_transactions!(node_block, local_block, inputs, outputs, outputs_da # locate correct record according to tx_hash binary_hashes = CkbUtils.hexes_to_bins_sql(hashes) pending_txs = CkbTransaction.where("tx_hash IN (#{binary_hashes})").where(tx_status: :pending).pluck( - :tx_hash, :created_at + :tx_hash, :confirmation_time ) CkbTransaction.where("tx_hash IN (#{binary_hashes})").update_all tx_status: "committed" @@ -1283,7 +1280,7 @@ def build_ckb_transactions!(node_block, local_block, inputs, outputs, outputs_da end.map do |tx| { id: tx["id"], tx_status: :committed, - confirmation_time: (tx["block_timestamp"].to_i / 1000) - hash_to_pool_times[tx["tx_hash"].tr("\\", "0")].to_i + confirmation_time: (tx["block_timestamp"].to_i - hash_to_pool_times[tx["tx_hash"].tr("\\", "0")].to_i) / 1000 } end CkbTransaction.upsert_all(confirmation_time_attrs, update_only: [:confirmation_time], @@ -1355,7 +1352,7 @@ def ckb_transaction_attributes(local_block, tx, tx_index) is_cellbase: tx_index.zero?, live_cell_changes: live_cell_changes(tx, tx_index), bytes: tx.serialized_size_in_block, - tx_index: tx_index + tx_index:, } end diff --git a/app/models/ckb_sync/transaction_parser.rb b/app/models/ckb_sync/transaction_parser.rb new file mode 100644 index 000000000..8e04dfa3d --- /dev/null +++ b/app/models/ckb_sync/transaction_parser.rb @@ -0,0 +1,151 @@ +module CkbSync + class TransactionParser + attr_reader :transaction, :extra_data + attr_accessor :tx_attr, :cell_outputs_attrs, :cell_data_attrs, :cell_inputs_attrs, :cell_deps_attrs, :witnesses_attrs, :header_deps_attrs, :lock_script_attrs, :addresses_attrs, + :account_books_attrs, :type_script_attrs + + # {"transaction"=>{"version"=>"0x0", "cell_deps"=>[{"out_point"=>{"tx_hash"=>"0xcd52d714ddea04d2917892f16d47cbd0bbbb7d9ba281233ec4021f79fc34bccc", "index"=>"0x0"}, "dep_type"=>"code"}, {"out_point"=>{"tx_hash"=>"0x9154df4f7336402114d04495175b37390ce86a4906d2d4001cf02c3e6d97f39c", "index"=>"0x0"}, "dep_type"=>"code"}, {"out_point"=>{"tx_hash"=>"0xbcd73881ba53f1cd95d0c855395c4ffe6f54e041765d9ab7602d48a7cb71612e", "index"=>"0x0"}, "dep_type"=>"code"}, {"out_point"=>{"tx_hash"=>"0xf8de3bb47d055cdf460d93a2a6e1b05f7432f9777c8c474abf4eec1d4aee5d37", "index"=>"0x0"}, "dep_type"=>"dep_group"}, {"out_point"=>{"tx_hash"=>"0x053fdb4ed3181eab3a3a5f05693b53a8cdec0a24569e16369f444bac48be7de9", "index"=>"0x0"}, "dep_type"=>"code"}], "header_deps"=>[], "inputs"=>[{"since"=>"0x40000000669f4e30", "previous_output"=>{"tx_hash"=>"0x3d9a919a18d2cc2b64d2063626c75a6c97e87d2e8c30ec7bd33ef3ce14039934", "index"=>"0x0"}}, {"since"=>"0x0", "previous_output"=>{"tx_hash"=>"0x3d9a919a18d2cc2b64d2063626c75a6c97e87d2e8c30ec7bd33ef3ce14039934", "index"=>"0x1"}}, {"since"=>"0x0", "previous_output"=>{"tx_hash"=>"0x3d9a919a18d2cc2b64d2063626c75a6c97e87d2e8c30ec7bd33ef3ce14039934", "index"=>"0x2"}}], "outputs"=>[{"capacity"=>"0x7676d7e00", "lock"=>{"code_hash"=>"0x79f90bb5e892d80dd213439eeab551120eb417678824f282b4ffb5f21bad2e1e", "hash_type"=>"type", "args"=>"0x00c267a8b93cdae15fb06325f11a72b1047bd4d33c00"}, "type"=>{"code_hash"=>"0x1e44736436b406f8e48a30dfbddcf044feb0c9eebfe63b0f81cb5bb727d84854", "hash_type"=>"type", "args"=>"0x86c7429247beba7ddd6e4361bcdfc0510b0b644131e2afb7e486375249a01802"}}, {"capacity"=>"0x3691d6afc000", "lock"=>{"code_hash"=>"0x7f5a09b8bd0e85bcf2ccad96411ccba2f289748a1c16900b0635c2ed9126f288", "hash_type"=>"type", "args"=>"0x702359ea7f073558921eb50d8c1c77e92f760c8f8656bde4995f26b8963e2dd8f245705db4fe72be953e4f9ee3808a1700a578341aa80a8b2349c236c4af64e5e077710000000000"}, "type"=>nil}, {"capacity"=>"0xe529edc1ba", "lock"=>{"code_hash"=>"0x9bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce8", "hash_type"=>"type", "args"=>"0xc267a8b93cdae15fb06325f11a72b1047bd4d33c"}, "type"=>nil}], "outputs_data"=>["0xa0cf6037bfc238b179b74a30a9b12e15a4fbdd8881aebc8e5a66a8b5b5c95f0a6d833f7d5c1f8130fa2688970b421a57fbdffeff55bae5b4676043c543355799abe60200de0991241ff42c0255e2e2f65d114951c0a144e89d35527c582adc6603ff1ea2e17771000000000000000000000000000000000000000000000000000000000000000000000000001f779faa1f6184b10c9d865f62bba000e5d54e00a5d4b98cd768e43e376f68421a6b49de900100007c777100000000000001", "0x", "0x"], "witnesses"=>["0xc1030000100000006900000069000000550000005500000010000000550000005500000041000000e74f7818a6d2d1dda76593b30973967cd15fc6853731ecc0a5cf42cdb81859005b4a22781fa67973ccfbd3c5d0c1ccdd62506e2e99a049c19adc972e3909511a0054030000000000005003000010000000480300004c030000380300001c0000006c01000070010000740100007801000034030000500100002c000000340000005400000074000000940000009c000000c0000000e4000000e80000000c010000e0777100000000001c0000000200000014000000715ab282b873b79a7be8b0e8c13c4e8966a52040f7cfb9cf096dc32d69cac2b6f884bb2b1a8bb01660f3edc613ccfbeb7f3506d6f245705db4fe72be953e4f9ee3808a1700a578341aa80a8b2349c236c4af64e51a6b49de900100006d833f7d5c1f8130fa2688970b421a57fbdffeff55bae5b4676043c543355799abe602006d833f7d5c1f8130fa2688970b421a57fbdffeff55bae5b4676043c543355799abe602000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000024ea893c4fa601a048b1d3a8de265fd8b442ba2a1ac37d85dfe320a7c8c2069a000000000000000004000000b80100004c4f05500d310a045d500ed295c313abef5ae2862c24393fdacc2e2e4c2ba76b43828b9d505cfc4cbb06c1083817aa14c6e06df9c300687a61fa0ec947fe79962fb9c557845058409c8c929c05fe19cc818cd0da6f0bf887cdb7ca4e465e85f7796ddd0e1528507119e4869f5a24e613de92605fb220451c0d1727d65f1c030815155c3cb7acd350a011fc6c115f153c24055880abd2a1253281e4e8a894e2a0546ea8799b057667507cee0dc2c4b8b171a0331b3f9ec02cf8906ef52a873523d2dbe3ff9e5f58699a4f01509d5c06f8f45bfaf59625237cb06c099a9c2a59752d5c1032e503ef39fbded2d7506989c3d61168d80ae0e27a2b2ca904f768cc698f783fd4f5d6c45c1b7dd66bbb5090d5dd9cd0b9f66197a69ea30e933982c98b99a067b37a8d0b7adc687d8c8db44f0150da2c4d3f8cc63b7b827c24c55098ec49cc88a9c81ff07abb0057f7d1dc05c9304f0350287fc2f5b5005ff558ae296f4e9cb354876829fa9562d215fbf7cd9536d5e0e550a03eb9492d55fc2d9d63faf8ea6121d66a86230f0752178ab5643a81abff8380508ad8ce2ac94cf885730b362ca4b81787bf85b48d7f72ef7816130bdc54433f644fe9040000000000000000000000", "0x10000000100000001000000010000000", "0x55000000100000005500000055000000410000001ccd614d2dbafb3384f6da8d50fc6cef21e0280c05397a93072d1560c81fd8bf347d299d7879e7b7682812fbb9fd5841c97c2f92b5c3e64737aa61c9fab7a01401"], "hash"=>"0x74758da9a59938724839e442d6e2b10b5c69e8fa398d4bd4d9b64fd311801ac1"}, "cycles"=>"0x4819ea", "size"=>"0x853", "fee"=>"0x853", "timestamp"=>"0x190de4b95b3"} + def initialize(tx, extra_data = {}) + @transaction = tx + @extra_data = extra_data + @cell_outputs_attrs = [] + @cell_data_attrs = [] + @cell_inputs_attrs = [] + @cell_deps_attrs = [] + @witnesses_attrs = [] + @header_deps_attrs = [] + @lock_script_attrs = Set.new + @addresses_attrs = Set.new + @account_books_attrs = Set.new + @type_script_attrs = Set.new + end + + def parse + prepare_transaction_params + prepare_cell_inputs_params + prepare_cell_outputs_params + prepare_cell_witness_params + prepare_header_deps_params + prepare_cell_deps_params + end + + def prepare_transaction_params + @tx_attr = + { + tx_hash: transaction.hash, + version: transaction.version, + tx_status: "pending", + transaction_fee: extra_data["fee"]&.hex, + bytes: extra_data["size"]&.hex, + capacity_involved: nil, + cycles: extra_data["cycles"]&.hex, + live_cell_changes: transaction.outputs.count - transaction.inputs.count, + confirmation_time: extra_data["timestamp"]&.hex, + } + end + + def prepare_cell_inputs_params + transaction.inputs.each_with_index do |input, index| + @cell_inputs_attrs << + { + since: input.since, + previous_tx_hash: input.previous_output.tx_hash, + previous_index: input.previous_output.index, + index:, + tx_hash: transaction.hash, + from_cell_base: input.previous_output.tx_hash == CellOutput::SYSTEM_TX_HASH, + block_id: nil, + cell_type: nil, + ckb_transaction_id: nil, + previous_cell_output_id: nil, + } + end + end + + def prepare_cell_outputs_params + transaction.outputs.each_with_index do |output, index| + output_data = transaction.outputs_data[index] + binary_data = CKB::Utils.hex_to_bin(output_data) + cell_type = CkbUtils.cell_type(output.type, output_data) + @lock_script_attrs << output.lock.to_h.merge({ script_hash: output.lock.compute_hash }) + @addresses_attrs << { address_hash: CkbUtils.generate_address(output.lock), lock_hash: output.lock.compute_hash } + @account_books_attrs << { lock_script_hash: output.lock.compute_hash, tx_hash: transaction.hash } + @type_script_attrs << output.type.to_h.merge({ script_hash: output.type.compute_hash }) if output.type.present? + @cell_outputs_attrs << + { + capacity: output.capacity, + tx_hash: transaction.hash, + cell_index: index, + status: "pending", + occupied_capacity: CkbUtils.cal_cell_min_capacity(output.lock, output.type, binary_data), + address_id: nil, + cell_type:, + lock_script_hash: output.lock.compute_hash, + type_hash: output.type&.compute_hash, + udt_amount: udt_amount(cell_type, output_data, output.type&.args), + data_size: binary_data.bytesize, + data_hash: CKB::Blake2b.hexdigest(binary_data), + block_id: nil, + block_timestamp: nil, + dao: nil, + } + + if output_data != "0x" + @cell_data_attrs << + { + data: output_data, + cell_index: index, + tx_hash: transaction.hash, + } + end + end + end + + def prepare_cell_witness_params + transaction.witnesses.each_with_index do |witness, index| + @witnesses_attrs << + { + data: witness, + index:, + tx_hash: transaction.hash, + } + end + end + + def prepare_header_deps_params + transaction.header_deps.each_with_index do |header_dep, index| + @header_deps_attrs << + { + header_hash: header_dep, + index:, + tx_hash: transaction.hash, + } + end + end + + def prepare_cell_deps_params + transaction.cell_deps.each do |cell_dep| + @cell_deps_attrs << + { + dep_type: cell_dep.dep_type, + out_point_tx_hash: cell_dep.out_point.tx_hash, + out_point_index: cell_dep.out_point.index, + tx_hash: transaction.hash, + } + end + end + + private + + def udt_amount(cell_type, output_data, type_script_args) + case cell_type + when "udt", "xudt", "xudt_compatible" + CkbUtils.parse_udt_cell_data(output_data) + when "omiga_inscription" + CkbUtils.parse_omiga_inscription_data(output_data)[:mint_limit] + when "m_nft_token" + "0x#{type_script_args[-8..]}".hex + end + end + end +end diff --git a/app/models/ckb_sync/transactions.rb b/app/models/ckb_sync/transactions.rb new file mode 100644 index 000000000..ef0360b01 --- /dev/null +++ b/app/models/ckb_sync/transactions.rb @@ -0,0 +1,128 @@ +module CkbSync + class Transactions + attr_accessor :parsers + + def initialize(raw_txs) + @parsers = + raw_txs.map do |raw_tx| + transaction, extra_data = + if raw_tx.is_a?(CKB::Types::Transaction) + [raw_tx, {}] + else + [CKB::Types::Transaction.from_h(raw_tx["transaction"].with_indifferent_access), raw_tx.except("transaction")] + end + + tx_parser = CkbSync::TransactionParser.new(transaction, extra_data) + tx_parser.parse + tx_parser + end + end + + def import + @parsers.each_slice(100).to_a.each do |group_parsers| + tx_attrs = [] + cell_outputs_attrs = [] + cell_data_attrs = [] + cell_inputs_attrs = [] + cell_deps_attrs = [] + witnesses_attrs = [] + header_deps_attrs = [] + lock_script_attrs = Set.new + addresses_attrs = Set.new + type_script_attrs = Set.new + account_books_attrs = Set.new + + group_parsers.each do |parser| + tx_attrs << parser.tx_attr + cell_outputs_attrs.concat(parser.cell_outputs_attrs) + cell_data_attrs.concat(parser.cell_data_attrs) + witnesses_attrs.concat(parser.witnesses_attrs) + cell_deps_attrs.concat(parser.cell_deps_attrs) + header_deps_attrs.concat(parser.header_deps_attrs) + cell_inputs_attrs.concat(parser.cell_inputs_attrs) + lock_script_attrs.merge(parser.lock_script_attrs) + addresses_attrs.merge(parser.addresses_attrs) + type_script_attrs.merge(parser.type_script_attrs) + account_books_attrs.merge(parser.account_books_attrs) + end + ApplicationRecord.transaction do + tx_returnings = CkbTransaction.upsert_all(tx_attrs, unique_by: %i[tx_status tx_hash], returning: %i[id tx_hash]) + tx_mappings = tx_returnings.rows.to_h { |id, tx_hash| [tx_hash.sub(/^\\x/, "0x"), id] } + lock_script_returnings = LockScript.upsert_all(lock_script_attrs.to_a, unique_by: :script_hash, returning: %i[id script_hash]) + lock_script_mappings = lock_script_returnings.rows.to_h { |id, script_hash| [script_hash, id] } + address_returnings = Address.upsert_all(addresses_attrs.to_a, unique_by: :lock_hash, returning: %i[id lock_hash]) + address_mappings = address_returnings.rows.to_h { |id, lock_hash| [[lock_hash.sub(/^\\x/, "")].pack("H*"), id] } + type_script_mappings = {} + if type_script_attrs.present? + type_script_returnings = TypeScript.upsert_all(type_script_attrs.to_a, unique_by: :script_hash, returning: %i[id script_hash]) + type_script_mappings = type_script_returnings.rows.to_h { |id, script_hash| [script_hash, id] } + end + new_account_books_attrs = + account_books_attrs.to_a.map do |attr| + { ckb_transaction_id: tx_mappings[attr[:tx_hash]], address_id: address_mappings[attr[:lock_script_hash]] } + end + AccountBook.upsert_all(new_account_books_attrs, unique_by: %i[address_id ckb_transaction_id]) + new_cell_outputs_attrs = + cell_outputs_attrs.map do |attr| + attr.merge({ ckb_transaction_id: tx_mappings[attr[:tx_hash]], lock_script_id: lock_script_mappings[attr[:lock_script_hash]], + type_script_id: type_script_mappings[attr[:type_hash]], address_id: address_mappings[attr[:lock_script_hash]] }).except(:lock_script_hash) + end + cell_outputs_returnings = CellOutput.upsert_all(new_cell_outputs_attrs, unique_by: %i[tx_hash cell_index status], returning: %i[id tx_hash cell_index]) + cell_output_mappings = cell_outputs_returnings.rows.to_h { |id, tx_hash, cell_index| ["#{tx_hash.sub(/^\\x/, '0x')}-#{cell_index}", id] } + new_cell_data_attrs = + cell_data_attrs.map do |attr| + { cell_output_id: cell_output_mappings["#{attr[:tx_hash]}-#{attr[:cell_index]}"], data: attr[:data] } + end + CellDatum.upsert_all(new_cell_data_attrs, unique_by: :cell_output_id) if new_cell_data_attrs.present? + new_witnesses_attrs = + witnesses_attrs.map do |attr| + { ckb_transaction_id: tx_mappings[attr[:tx_hash]], data: attr[:data], index: attr[:index] } + end + Witness.upsert_all(new_witnesses_attrs, unique_by: %i[ckb_transaction_id index]) + new_header_deps_attrs = + header_deps_attrs.map do |attr| + { ckb_transaction_id: tx_mappings[attr[:tx_hash]], header_hash: attr[:header_hash], index: attr[:index] } + end + HeaderDependency.upsert_all(new_header_deps_attrs, unique_by: %i[ckb_transaction_id index]) if new_header_deps_attrs.present? + cell_dep_conditions = cell_deps_attrs.map { |cell_dep| { tx_hash: cell_dep[:out_point_tx_hash], cell_index: cell_dep[:out_point_index] } } + cell_dep_returnings = batch_query_outputs(cell_dep_conditions, %i[id tx_hash cell_index]) + cell_dep_mappings = cell_dep_returnings.to_h { |id, tx_hash, cell_index| ["#{tx_hash}-#{cell_index}", id] } + new_cell_deps_attrs = + cell_deps_attrs.map do |attr| + { ckb_transaction_id: tx_mappings[attr[:tx_hash]], contract_cell_id: cell_dep_mappings["#{attr[:out_point_tx_hash]}-#{attr[:out_point_index]}"], dep_type: attr[:dep_type] } + end.filter { |attr| !attr[:contract_cell_id].nil? } + CellDependency.upsert_all(new_cell_deps_attrs, unique_by: %i[ckb_transaction_id contract_cell_id]) if new_cell_deps_attrs.present? + + input_conditions = cell_inputs_attrs.filter do |input| + input[:previous_tx_hash] != CellOutput::SYSTEM_TX_HASH + end.map { |input| { tx_hash: input[:previous_tx_hash], cell_index: input[:previous_index] } } + input_returnings = batch_query_outputs(input_conditions, %i[id cell_type tx_hash cell_index capacity]) + input_mappings = input_returnings.to_h { |id, cell_type, tx_hash, cell_index| ["#{tx_hash}-#{cell_index}", "#{cell_type}-#{id}"] } + new_cell_inputs_attrs = + cell_inputs_attrs.map do |attr| + attr[:ckb_transaction_id] = tx_mappings[attr[:tx_hash]] + if attr[:previous_tx_hash] != CellOutput::SYSTEM_TX_HASH && input_mappings["#{attr[:previous_tx_hash]}-#{attr[:previous_index]}"].present? + cell_type, previous_cell_output_id = input_mappings["#{attr[:previous_tx_hash]}-#{attr[:previous_index]}"].split("-") + attr[:previous_cell_output_id] = previous_cell_output_id + attr[:cell_type] = cell_type + end + attr.except(:tx_hash) + end + CellInput.upsert_all(new_cell_inputs_attrs, unique_by: %i[ckb_transaction_id index]) + end + end + end + + private + + def batch_query_outputs(conditions, returnings = %i[id cell_type tx_hash cell_index]) + relation = CellOutput.none + + conditions.each do |condition| + relation = relation.or(CellOutput.where(condition)) + end + + relation.pluck(Arel.sql(returnings.join(", "))) + end + end +end diff --git a/app/models/ckb_transaction.rb b/app/models/ckb_transaction.rb index e9435a936..cf89344c1 100644 --- a/app/models/ckb_transaction.rb +++ b/app/models/ckb_transaction.rb @@ -274,7 +274,7 @@ def log_deletion_chain # tags :string default([]), is an Array # bytes :bigint default(0) # cycles :bigint -# confirmation_time :integer +# confirmation_time :bigint # tx_index :integer # # Indexes diff --git a/app/models/lock_script.rb b/app/models/lock_script.rb index 77032f25e..76cc760c7 100644 --- a/app/models/lock_script.rb +++ b/app/models/lock_script.rb @@ -150,6 +150,6 @@ def tip_epoch(tip_block) # index_lock_scripts_on_address_id (address_id) # index_lock_scripts_on_cell_output_id (cell_output_id) # index_lock_scripts_on_code_hash_and_hash_type_and_args (code_hash,hash_type,args) -# index_lock_scripts_on_script_hash (script_hash) USING hash +# index_lock_scripts_on_script_hash (script_hash) UNIQUE # index_lock_scripts_on_script_id (script_id) # diff --git a/app/models/type_script.rb b/app/models/type_script.rb index d4705291e..bc03e9184 100644 --- a/app/models/type_script.rb +++ b/app/models/type_script.rb @@ -81,6 +81,6 @@ def calculate_bytesize # # index_type_scripts_on_cell_output_id (cell_output_id) # index_type_scripts_on_code_hash_and_hash_type_and_args (code_hash,hash_type,args) -# index_type_scripts_on_script_hash (script_hash) USING hash +# index_type_scripts_on_script_hash (script_hash) UNIQUE # index_type_scripts_on_script_id (script_id) # diff --git a/app/utils/ckb_utils.rb b/app/utils/ckb_utils.rb index 96b423d92..125ea5f1a 100644 --- a/app/utils/ckb_utils.rb +++ b/app/utils/ckb_utils.rb @@ -11,10 +11,7 @@ def self.calculate_cell_min_capacity(output, data) output.calculate_min_capacity(data) end - def self.cal_cell_min_capacity(lock_script, type_script, capacity, binary_data) - lock = CKB::Types::Script.new(**lock_script.to_node) - type = type_script.present? ? CKB::Types::Script.new(**type_script.to_node) : nil - CKB::Types::Output.new(capacity: capacity.to_i, lock:, type:) + def self.cal_cell_min_capacity(lock_script, type_script, binary_data) CKB::Utils.byte_to_shannon([8, binary_data&.bytesize || 0, lock_script.calculate_bytesize, type_script&.calculate_bytesize || 0].sum) end diff --git a/app/workers/import_pending_tx_worker.rb b/app/workers/import_pending_tx_worker.rb new file mode 100644 index 000000000..8130fe4d4 --- /dev/null +++ b/app/workers/import_pending_tx_worker.rb @@ -0,0 +1,15 @@ +# Check every pending transaction in the pool if rejected +class ImportPendingTxWorker + include Sidekiq::Worker + sidekiq_options retry: 1, queue: "pending_tx" + + def perform(data) + committed_tx = CkbTransaction.tx_committed.find_by(tx_hash: data["transaction"]["hash"]) + if committed_tx && committed_tx.confirmation_time.nil? + confirmation_time = committed_tx.block_timestamp - data["timestamp"].hex + committed_tx.update(confirmation_time:) + else + CkbSync::Transactions.new([data]).import + end + end +end diff --git a/config/database.yml b/config/database.yml index 2405ef716..1ef3510ca 100644 --- a/config/database.yml +++ b/config/database.yml @@ -27,7 +27,7 @@ default: &default reaping_frequency: 60 # every 60s check pool timeout: 5000 # over 5s will timeout variables: - statement_timeout: 60000 # query timemout over 60s + statement_timeout: <%= ENV.fetch("DATABASE_STATEMENT_TIMEOUT") { 60000 } %> # query timemout over 60s idle_in_transaction_session_timeout: 120000 # idle transaction over 120s development: diff --git a/config/sidekiq.yml b/config/sidekiq.yml index 226b22ba6..eb0df0b76 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -1,6 +1,7 @@ --- :concurrency: 5 :queues: + - pending_tx - critical - default - bitcoin diff --git a/db/migrate/20240823071323_add_unique_index_to_lock_script.rb b/db/migrate/20240823071323_add_unique_index_to_lock_script.rb new file mode 100644 index 000000000..08611666f --- /dev/null +++ b/db/migrate/20240823071323_add_unique_index_to_lock_script.rb @@ -0,0 +1,13 @@ +class AddUniqueIndexToLockScript < ActiveRecord::Migration[7.0] + def up + # set 1 hour + execute "SET statement_timeout = 3600000" + + remove_index :lock_scripts, :script_hash + add_index :lock_scripts, :script_hash, unique: true + end + + def down + execute "RESET statement_timeout" + end +end diff --git a/db/migrate/20240823071420_add_unique_index_to_type_script.rb b/db/migrate/20240823071420_add_unique_index_to_type_script.rb new file mode 100644 index 000000000..e7650a460 --- /dev/null +++ b/db/migrate/20240823071420_add_unique_index_to_type_script.rb @@ -0,0 +1,13 @@ +class AddUniqueIndexToTypeScript < ActiveRecord::Migration[7.0] + def up + # set 1 hour + execute "SET statement_timeout = 3600000" + + remove_index :type_scripts, :script_hash + add_index :type_scripts, :script_hash, unique: true + end + + def down + execute "RESET statement_timeout" + end +end diff --git a/db/migrate/20240904043807_change_ckb_transaction_confirmation_time.rb b/db/migrate/20240904043807_change_ckb_transaction_confirmation_time.rb new file mode 100644 index 000000000..c6a8ad074 --- /dev/null +++ b/db/migrate/20240904043807_change_ckb_transaction_confirmation_time.rb @@ -0,0 +1,6 @@ +class ChangeCkbTransactionConfirmationTime < ActiveRecord::Migration[7.0] + def change + ActiveRecord::Base.connection.execute("SET statement_timeout = 0") + change_column :ckb_transactions, :confirmation_time, :bigint + end +end diff --git a/db/structure.sql b/db/structure.sql index 8364d22af..b0ee52b1a 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -1293,7 +1293,7 @@ CREATE TABLE public.ckb_transactions ( tags character varying[] DEFAULT '{}'::character varying[], bytes bigint DEFAULT 0, cycles bigint, - confirmation_time integer, + confirmation_time bigint, tx_index integer ) PARTITION BY LIST (tx_status); @@ -1339,7 +1339,7 @@ CREATE TABLE public.ckb_transactions_committed ( tags character varying[] DEFAULT '{}'::character varying[], bytes bigint DEFAULT 0, cycles bigint, - confirmation_time integer, + confirmation_time bigint, tx_index integer ); @@ -1365,7 +1365,7 @@ CREATE TABLE public.ckb_transactions_pending ( tags character varying[] DEFAULT '{}'::character varying[], bytes bigint DEFAULT 0, cycles bigint, - confirmation_time integer, + confirmation_time bigint, tx_index integer ); @@ -1391,7 +1391,7 @@ CREATE TABLE public.ckb_transactions_proposed ( tags character varying[] DEFAULT '{}'::character varying[], bytes bigint DEFAULT 0, cycles bigint, - confirmation_time integer, + confirmation_time bigint, tx_index integer ); @@ -1417,7 +1417,7 @@ CREATE TABLE public.ckb_transactions_rejected ( tags character varying[] DEFAULT '{}'::character varying[], bytes bigint DEFAULT 0, cycles bigint, - confirmation_time integer, + confirmation_time bigint, tx_index integer ); @@ -4788,7 +4788,7 @@ CREATE INDEX index_lock_scripts_on_code_hash_and_hash_type_and_args ON public.lo -- Name: index_lock_scripts_on_script_hash; Type: INDEX; Schema: public; Owner: - -- -CREATE INDEX index_lock_scripts_on_script_hash ON public.lock_scripts USING hash (script_hash); +CREATE UNIQUE INDEX index_lock_scripts_on_script_hash ON public.lock_scripts USING btree (script_hash); -- @@ -4998,7 +4998,7 @@ CREATE INDEX index_type_scripts_on_code_hash_and_hash_type_and_args ON public.ty -- Name: index_type_scripts_on_script_hash; Type: INDEX; Schema: public; Owner: - -- -CREATE INDEX index_type_scripts_on_script_hash ON public.type_scripts USING hash (script_hash); +CREATE UNIQUE INDEX index_type_scripts_on_script_hash ON public.type_scripts USING btree (script_hash); -- @@ -5942,6 +5942,7 @@ INSERT INTO "schema_migrations" (version) VALUES ('20240709131713'), ('20240709142013'), ('20240822024448'), -('20240902025657'); - - +('20240823071323'), +('20240823071420'), +('20240902025657'), +('20240904043807'); diff --git a/lib/websocket.rb b/lib/websocket.rb index 3bff70e1f..7d124ed41 100644 --- a/lib/websocket.rb +++ b/lib/websocket.rb @@ -22,36 +22,36 @@ def subscribe(connection, topic) $message_id += 1 message = Protocol::WebSocket::JSONMessage.generate({ - "id": $message_id, - "jsonrpc": "2.0", - "method": "subscribe", - "params": [topic] - }) + "id": $message_id, + "jsonrpc": "2.0", + "method": "subscribe", + "params": [topic], + }) message.send(connection) connection.flush end -queue = Queue.new +# queue = Queue.new -persister = - Thread.new do - Rails.application.executor.wrap do - loop do - data = queue.pop +# persister = +# Thread.new do +# Rails.application.executor.wrap do +# loop do +# data = queue.pop - begin - ImportTransactionJob.new.perform(data["transaction"], { - cycles: data["cycles"].hex, - fee: data["fee"].hex, - size: data["size"].hex, - timestamp: data["timestamp"].hex - }) - rescue StandardError => e - Rails.logger.error "Error occurred during ImportTransactionJob data: #{data}, error: #{e.message}" - end - end - end - end +# begin +# ImportTransactionJob.new.perform(data["transaction"], { +# cycles: data["cycles"].hex, +# fee: data["fee"].hex, +# size: data["size"].hex, +# timestamp: data["timestamp"].hex, +# }) +# rescue StandardError => e +# Rails.logger.error "Error occurred during ImportTransactionJob data: #{data}, error: #{e.message}" +# end +# end +# end +# end Async do |_task| endpoint = Async::HTTP::Endpoint.parse(URL, alpn_protocols: Async::HTTP::Protocol::HTTP11.names) @@ -64,9 +64,7 @@ def subscribe(connection, topic) res = message.to_h if res[:method] == "subscribe" data = JSON.parse res[:params][:result] - # binding.pry - puts data["transaction"]["hash"] - queue.push(data) + ImportPendingTxWorker.perform_async(data) end end end diff --git a/test/factories/type_script.rb b/test/factories/type_script.rb index 597c3c238..db42c27a5 100644 --- a/test/factories/type_script.rb +++ b/test/factories/type_script.rb @@ -7,5 +7,6 @@ code_hash { "0x00000000000000000000000000000000000000000000000000545950455f4944" } hash_type { "type" } end + after(:build) { |ts, _context| ts.script_hash = CKB::Types::Script.new(code_hash: ts.code_hash, args: ts.args, hash_type: ts.hash_type).compute_hash } end end diff --git a/test/models/ckb_sync/node_data_processor_test.rb b/test/models/ckb_sync/node_data_processor_test.rb index 683ec346a..f9b790284 100644 --- a/test/models/ckb_sync/node_data_processor_test.rb +++ b/test/models/ckb_sync/node_data_processor_test.rb @@ -148,8 +148,9 @@ class NodeDataProcessorTest < ActiveSupport::TestCase ), ) VCR.use_cassette("blocks/11") do + timestamp = Time.now.to_i * 1000 tx = create(:pending_transaction, - tx_hash: "0x4298daf91148df9093c844d2ae7d16bee6b74e7ab1ccccd108ce834d1ca1a56c") + tx_hash: "0x4298daf91148df9093c844d2ae7d16bee6b74e7ab1ccccd108ce834d1ca1a56c", confirmation_time: timestamp) node_block = CkbSync::Api.instance.get_block_by_number(11) create(:block, :with_block_hash, number: node_block.header.number - 1) node_block.transactions.first.hash = tx.tx_hash @@ -159,7 +160,7 @@ class NodeDataProcessorTest < ActiveSupport::TestCase node_data_processor.process_block(node_block) end assert_equal tx.reload.confirmation_time, - tx.reload.block_timestamp.to_i / 1000 - tx.created_at.to_i + (tx.reload.block_timestamp.to_i - timestamp) / 1000 end end @@ -2839,14 +2840,8 @@ class NodeDataProcessorTest < ActiveSupport::TestCase create(:type_script, args: udt_script.args, code_hash: Settings.sudt_cell_type_hash, hash_type: "data", cell_output: output1) - create(:type_script, args: udt_script.args, code_hash: Settings.sudt_cell_type_hash, hash_type: "data", - cell_output: output2) - create(:type_script, args: udt_script.args, code_hash: Settings.sudt_cell_type_hash, hash_type: "data", - cell_output: output3) create(:type_script, args: udt_script1.args, code_hash: Settings.sudt_cell_type_hash, hash_type: "data", cell_output: output4) - create(:type_script, args: udt_script1.args, code_hash: Settings.sudt_cell_type_hash, hash_type: "data", - cell_output: output5) Address.create(lock_hash: udt_script.args, address_hash: "0x#{SecureRandom.hex(32)}") Address.create(lock_hash: udt_script1.args, @@ -3112,15 +3107,6 @@ class NodeDataProcessorTest < ActiveSupport::TestCase create(:type_script, args: udt_script.args, code_hash: Settings.sudt_cell_type_hash, hash_type: "data") - create(:type_script, args: udt_script.args, - code_hash: Settings.sudt_cell_type_hash, - hash_type: "data") - create(:type_script, args: udt_script.args, - code_hash: Settings.sudt_cell_type_hash, - hash_type: "data") - create(:type_script, args: udt_script1.args, - code_hash: Settings.sudt_cell_type_hash, - hash_type: "data") create(:type_script, args: udt_script1.args, code_hash: Settings.sudt_cell_type_hash, hash_type: "data") @@ -3434,12 +3420,6 @@ class NodeDataProcessorTest < ActiveSupport::TestCase tx_hash: tx4.tx_hash, cell_index: 0, address: input_address4, cell_type: "udt", lock_script_id: address4_lock.id, type_script_id: type_script1.id, type_hash: udt_script1.compute_hash) output5 = create(:cell_output, ckb_transaction: tx5, block: block2, capacity: 70000 * 10**8, tx_hash: tx5.tx_hash, cell_index: 0, address: input_address5, cell_type: "udt", lock_script_id: address5_lock.id, type_script_id: type_script2.id, type_hash: udt_script2.compute_hash) - create(:type_script, args: udt_script1.args, code_hash: Settings.sudt_cell_type_hash, hash_type: "data", - cell_output: output1) - create(:type_script, args: udt_script1.args, code_hash: Settings.sudt_cell_type_hash, hash_type: "data", - cell_output: output2) - create(:type_script, args: udt_script2.args, code_hash: Settings.sudt_cell_type_hash, hash_type: "data", - cell_output: output3) output1.update(type_hash: CKB::Types::Script.new(**output1.type_script.to_node).compute_hash) output2.update(type_hash: CKB::Types::Script.new(**output2.type_script.to_node).compute_hash) output3.update(type_hash: CKB::Types::Script.new(**output3.type_script.to_node).compute_hash) @@ -3449,7 +3429,6 @@ class NodeDataProcessorTest < ActiveSupport::TestCase address_hash: "0x#{SecureRandom.hex(32)}") Address.create(lock_hash: udt_script2.args, address_hash: "0x#{SecureRandom.hex(32)}") - header = CKB::Types::BlockHeader.new(compact_target: "0x1000", hash: "0x#{SecureRandom.hex(32)}", number: DEFAULT_NODE_BLOCK_NUMBER, parent_hash: "0x#{SecureRandom.hex(32)}", nonce: 1757392074788233522, timestamp: CkbUtils.time_in_milliseconds(Time.current), transactions_root: "0x#{SecureRandom.hex(32)}", proposals_hash: "0x#{SecureRandom.hex(32)}", extra_hash: "0x#{SecureRandom.hex(32)}", version: 0, epoch: 1, dao: "0x01000000000000000000c16ff286230000a3a65e97fd03000057c138586f0000") inputs = [ @@ -3643,12 +3622,6 @@ class NodeDataProcessorTest < ActiveSupport::TestCase hash_type: "data") type_script2 = create(:type_script, args: udt_script2.args, code_hash: Settings.sudt_cell_type_hash, hash_type: "data") - type_script3 = create(:type_script, args: udt_script1.args, code_hash: Settings.sudt_cell_type_hash, - hash_type: "data") - type_script4 = create(:type_script, args: udt_script1.args, code_hash: Settings.sudt_cell_type_hash, - hash_type: "data") - type_script5 = create(:type_script, args: udt_script2.args, code_hash: Settings.sudt_cell_type_hash, - hash_type: "data") udt1 = create(:udt, type_hash: CKB::Types::Script.new(**type_script1.to_node).compute_hash, args: udt_script1.args, ckb_transactions_count: 3) udt2 = create(:udt, type_hash: CKB::Types::Script.new(**type_script2.to_node).compute_hash, @@ -3688,11 +3661,11 @@ class NodeDataProcessorTest < ActiveSupport::TestCase output2 = create(:cell_output, ckb_transaction: tx2, block: block2, capacity: 60000 * 10**8, tx_hash: tx2.tx_hash, cell_index: 1, address: input_address2, cell_type: "udt", lock_script_id: address2_lock.id, type_script_id: type_script2.id) output3 = create(:cell_output, ckb_transaction: tx3, block: block2, capacity: 70000 * 10**8, - tx_hash: tx3.tx_hash, cell_index: 2, address: input_address3, cell_type: "udt", lock_script_id: address3_lock.id, type_script_id: type_script3.id) + tx_hash: tx3.tx_hash, cell_index: 2, address: input_address3, cell_type: "udt", lock_script_id: address3_lock.id, type_script_id: type_script1.id) output4 = create(:cell_output, ckb_transaction: tx4, block: block2, capacity: 70000 * 10**8, - tx_hash: tx4.tx_hash, cell_index: 0, address: input_address4, cell_type: "udt", lock_script_id: address4_lock.id, type_script_id: type_script4.id) + tx_hash: tx4.tx_hash, cell_index: 0, address: input_address4, cell_type: "udt", lock_script_id: address4_lock.id, type_script_id: type_script1.id) output5 = create(:cell_output, ckb_transaction: tx5, block: block2, capacity: 70000 * 10**8, - tx_hash: tx5.tx_hash, cell_index: 0, address: input_address5, cell_type: "udt", lock_script_id: address5_lock.id, type_script_id: type_script5.id) + tx_hash: tx5.tx_hash, cell_index: 0, address: input_address5, cell_type: "udt", lock_script_id: address5_lock.id, type_script_id: type_script2.id) output1.update(type_hash: CKB::Types::Script.new(**output1.type_script.to_node).compute_hash) output2.update(type_hash: CKB::Types::Script.new(**output2.type_script.to_node).compute_hash) output3.update(type_hash: CKB::Types::Script.new(**output3.type_script.to_node).compute_hash) @@ -3960,9 +3933,8 @@ class NodeDataProcessorTest < ActiveSupport::TestCase hash_type: "type", args: old_factory_cell.args, ) - type_script1 = create(:type_script, args: factory_cell_script.args, - code_hash: factory_cell_script.code_hash, - hash_type: "type") + type_script1 = TypeScript.find_by(script_hash: factory_cell_script.compute_hash) + block1 = create(:block, :with_block_hash, number: DEFAULT_NODE_BLOCK_NUMBER - 1) tx1 = create(:ckb_transaction, block: block1) diff --git a/test/models/ckb_transaction_test.rb b/test/models/ckb_transaction_test.rb index e7c5a1eb6..dd8333a99 100644 --- a/test/models/ckb_transaction_test.rb +++ b/test/models/ckb_transaction_test.rb @@ -685,25 +685,20 @@ class CkbTransactionTest < ActiveSupport::TestCase nrc_721_token_output_block = create(:block, :with_block_hash) nrc_721_token_output_transaction = create(:ckb_transaction, block: nrc_721_token_output_block) - nrc_factory_cell = create(:nrc_factory_cell, code_hash: "0xf31f9c3241043976c93db9f203d765ce2c1d96465c918b93fbb117cd35bcfb98", args: "0x47a6262383ad9b266c6fae7c460dc39261e4f914e77467426809ce0fd607baec", verified: true) - nrc_721_factory_type_script = create(:type_script, - code_hash: nrc_factory_cell.code_hash, - hash_type: "type", - args: nrc_factory_cell.args, - script_hash: "0xcf4f2494ea1fbebcd07a7445415a0e8b36be214af6257065888dae0ce1434434") - nrc_721_factory_cell_output = create(:cell_output, - block: nrc_721_token_output_block, - ckb_transaction: nrc_721_token_output_transaction, - cell_type: "nrc_721_factory", - cell_index: 1, - tx_hash: nrc_721_token_output_transaction.tx_hash, - data: "0x24ff5a9ab8c38d195ce2b4ea75ca8987000a47616d62697420317374000000156465762e6b6f6c6c6563742e6d652f746f6b656e73000000000000003c000000000000000000", - type_hash: "0x", - type_script_id: nrc_721_factory_type_script.id) + nrc_721_factory_type_script = TypeScript.find_by(script_hash: "0xcf4f2494ea1fbebcd07a7445415a0e8b36be214af6257065888dae0ce1434434") + _nrc_721_factory_cell_output = create(:cell_output, + block: nrc_721_token_output_block, + ckb_transaction: nrc_721_token_output_transaction, + cell_type: "nrc_721_factory", + cell_index: 1, + tx_hash: nrc_721_token_output_transaction.tx_hash, + data: "0x24ff5a9ab8c38d195ce2b4ea75ca8987000a47616d62697420317374000000156465762e6b6f6c6c6563742e6d652f746f6b656e73000000000000003c000000000000000000", + type_hash: "0x", + type_script_id: nrc_721_factory_type_script.id) nrc_721_token_type_script = create(:type_script, code_hash: "0x#{SecureRandom.hex(32)}", diff --git a/test/utils/ckb_utils_test.rb b/test/utils/ckb_utils_test.rb index b21bc7498..7b0c082f7 100644 --- a/test/utils/ckb_utils_test.rb +++ b/test/utils/ckb_utils_test.rb @@ -209,7 +209,7 @@ class CkbUtilsTest < ActiveSupport::TestCase lock_script = create(:lock_script, code_hash: "0x9bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce8", hash_type: "type", args: "0x6a21bc1b72d1e654f8e2ded400cffa46075494c6") type_script = create(:type_script, code_hash: "0x554cff969f3148e3c620749384004e9692e67c429f621554d139b505a281c7b8", hash_type: "type", args: "0x01") assert_equal 9700000000, - CkbUtils.cal_cell_min_capacity(lock_script, type_script, 0.4e11, "\n\f") + CkbUtils.cal_cell_min_capacity(lock_script, type_script, "\n\f") end test ".block_cell_consumed generated block's cell_consumed should equal to the sum of transactions output occupied capacity" do