From a58c1d9aee8b69ec2fbf43f5ceefb3f66ac7e90c Mon Sep 17 00:00:00 2001 From: Rabbit Date: Tue, 30 Apr 2024 13:22:30 +0800 Subject: [PATCH 1/8] refactor: bitcoin transaction detect worker (#1825) * refactor: bitcoin transaction detect worker * chore: fix typo * feat: generate bitcoin transfers * chore: update structure.sql --- .../api/v2/bitcoin_transactions_controller.rb | 2 +- app/jobs/import_btc_time_cell_job.rb | 59 ++++++++ ...n_utxo_job.rb => import_rgbpp_cell_job.rb} | 68 +++++++-- app/models/bitcoin_address_mapping.rb | 2 +- app/models/bitcoin_transaction.rb | 1 + app/models/bitcoin_transfer.rb | 26 ++++ .../ckb_sync/new_node_data_processor.rb | 2 +- .../bitcoin_transaction_detect_worker.rb | 118 ++++++++++----- ...ename_index_of_bitcoin_address_mappings.rb | 5 + ...20240429102325_create_bitcoin_transfers.rb | 16 ++ db/structure.sql | 143 +++++++++++++++++- .../migration/generate_bitcoin_transfers.rake | 34 +++++ test/factories/bitcoin_transfers.rb | 5 + test/models/bitcoin_transfer_test.rb | 7 + 14 files changed, 419 insertions(+), 69 deletions(-) create mode 100644 app/jobs/import_btc_time_cell_job.rb rename app/jobs/{import_bitcoin_utxo_job.rb => import_rgbpp_cell_job.rb} (63%) create mode 100644 app/models/bitcoin_transfer.rb create mode 100644 db/migrate/20240428085020_rename_index_of_bitcoin_address_mappings.rb create mode 100644 db/migrate/20240429102325_create_bitcoin_transfers.rb create mode 100644 lib/tasks/migration/generate_bitcoin_transfers.rake create mode 100644 test/factories/bitcoin_transfers.rb create mode 100644 test/models/bitcoin_transfer_test.rb diff --git a/app/controllers/api/v2/bitcoin_transactions_controller.rb b/app/controllers/api/v2/bitcoin_transactions_controller.rb index 2b14ad10a..252866eef 100644 --- a/app/controllers/api/v2/bitcoin_transactions_controller.rb +++ b/app/controllers/api/v2/bitcoin_transactions_controller.rb @@ -22,7 +22,7 @@ def query render json: res rescue StandardError => e - Rails.logger.error "get raw transaction(#{params[:txids]}) failed: #{e.message}" + Rails.logger.error "get raw transactions(#{params[:txids]}) failed: #{e.message}" render json: {}, status: :not_found end diff --git a/app/jobs/import_btc_time_cell_job.rb b/app/jobs/import_btc_time_cell_job.rb new file mode 100644 index 000000000..f22ac5d95 --- /dev/null +++ b/app/jobs/import_btc_time_cell_job.rb @@ -0,0 +1,59 @@ +class ImportBtcTimeCellJob < ApplicationJob + queue_as :bitcoin + + def perform(cell_id) + ApplicationRecord.transaction do + cell_output = CellOutput.find_by(id: cell_id) + return unless cell_output + + lock_script = cell_output.lock_script + return unless CkbUtils.is_btc_time_lock_cell?(lock_script) + + parsed_args = CkbUtils.parse_btc_time_lock_cell(lock_script.args) + Rails.logger.info("Importing btc time cell txid #{parsed_args.txid}") + + # build bitcoin transaction + raw_tx = fetch_raw_transaction(txid) + return unless raw_tx + + tx = build_transaction!(raw_tx) + # build transfer + BitcoinTransfer.create_with( + bitcoin_transaction_id: tx.id, + ckb_transaction_id: cell_output.ckb_transaction_id, + lock_type: "btc_time", + ).find_or_create_by!( + cell_output_id: cell_id, + ) + end + end + + def build_transaction!(raw_tx) + tx = BitcoinTransaction.find_by(txid: raw_tx["txid"]) + return tx if tx + + BitcoinTransaction.create!( + txid: raw_tx["txid"], + tx_hash: raw_tx["hash"], + time: raw_tx["time"], + block_hash: raw_tx["blockhash"], + block_height: 0, + ) + end + + def fetch_raw_transaction(txid) + data = Rails.cache.read(txid) + data ||= rpc.getrawtransaction(txid, 2) + + return if data && data["error"].present? + + Rails.cache.write(txid, data, expires_in: 10.minutes) unless Rails.cache.exist?(txid) + data + rescue StandardError => e + raise ArgumentError, "get bitcoin raw transaction #{txid} failed: #{e}" + end + + def rpc + @rpc ||= Bitcoin::Rpc.instance + end +end diff --git a/app/jobs/import_bitcoin_utxo_job.rb b/app/jobs/import_rgbpp_cell_job.rb similarity index 63% rename from app/jobs/import_bitcoin_utxo_job.rb rename to app/jobs/import_rgbpp_cell_job.rb index c383361b5..2c522e421 100644 --- a/app/jobs/import_bitcoin_utxo_job.rb +++ b/app/jobs/import_rgbpp_cell_job.rb @@ -1,34 +1,53 @@ -class ImportBitcoinUtxoJob < ApplicationJob +class ImportRgbppCellJob < ApplicationJob queue_as :bitcoin def perform(cell_id) ApplicationRecord.transaction do cell_output = CellOutput.find_by(id: cell_id) - unless cell_output - raise ArgumentError, "Missing cell_output(#{cell_id})" - end + return unless cell_output lock_script = cell_output.lock_script return unless CkbUtils.is_rgbpp_lock_cell?(lock_script) txid, out_index = CkbUtils.parse_rgbpp_args(lock_script.args) - Rails.logger.info("Importing bitcoin utxo #{txid} out_index #{out_index}") - vout_attributes = [] + Rails.logger.info("Importing rgbpp cell txid #{txid} out_index #{out_index}") + # build bitcoin transaction - raw_tx = rpc.getrawtransaction(txid, 2) + raw_tx = fetch_raw_transaction(txid) + return unless raw_tx + tx = build_transaction!(raw_tx) # build op_returns - op_returns = build_op_returns!(raw_tx, tx, cell_output.ckb_transaction, vout_attributes) + vout_attributes = [] + op_returns = build_op_returns!(raw_tx, tx, cell_output.ckb_transaction) vout_attributes.concat(op_returns) if op_returns.present? - # build vout + # build vouts vout_attributes << build_vout!(raw_tx, tx, out_index, cell_output) - if vout_attributes.present? BitcoinVout.upsert_all( vout_attributes, unique_by: %i[bitcoin_transaction_id index cell_output_id], ) end + # build vin + cell_input = CellInput.find_by(previous_cell_output_id: cell_id) + previous_vout = BitcoinVout.find_by(cell_output_id: cell_id) + if cell_input && previous_vout + BitcoinVin.create_with( + previous_bitcoin_vout_id: previous_vout.id, + ).find_or_create_by!( + ckb_transaction_id: cell_input.ckb_transaction_id, + cell_input_id: cell_input.id, + ) + end + # build transfer + BitcoinTransfer.create_with( + bitcoin_transaction_id: tx.id, + ckb_transaction_id: cell_output.ckb_transaction_id, + lock_type: "rgbpp", + ).find_or_create_by!( + cell_output_id: cell_id, + ) end end @@ -49,7 +68,7 @@ def build_transaction!(raw_tx) ) end - def build_op_returns!(raw_tx, tx, ckb_tx, v_attributes) + def build_op_returns!(raw_tx, tx, ckb_tx) op_returns = [] raw_tx["vout"].each do |vout| @@ -57,10 +76,10 @@ def build_op_returns!(raw_tx, tx, ckb_tx, v_attributes) script_pubkey = Bitcoin::Script.parse_from_payload(data.htb) next unless script_pubkey.op_return? - # commiment = script_pubkey.op_return_data.bth - # unless commiment == CkbUtils.calculate_commitment(ckb_tx.tx_hash) - # raise ArgumentError, "Invalid commitment found in the CKB VirtualTx" - # end + # commiment = script_pubkey.op_return_data.bth + # unless commiment == CkbUtils.calculate_commitment(ckb_tx.tx_hash) + # raise ArgumentError, "Invalid commitment found in the CKB VirtualTx" + # end op_return = { bitcoin_transaction_id: tx.id, @@ -74,7 +93,12 @@ def build_op_returns!(raw_tx, tx, ckb_tx, v_attributes) address_id: nil, } - op_returns << op_return if v_attributes.exclude?(op_return) + next if BitcoinVout.exists?( + bitcoin_transaction_id: op_return[:bitcoin_transaction_id], + index: op_return[:index], + ) + + op_returns << op_return end op_returns @@ -110,6 +134,18 @@ def build_address!(address_hash, cell_output) bitcoin_address end + def fetch_raw_transaction(txid) + data = Rails.cache.read(txid) + data ||= rpc.getrawtransaction(txid, 2) + + return if data && data["error"].present? + + Rails.cache.write(txid, data, expires_in: 10.minutes) unless Rails.cache.exist?(txid) + data + rescue StandardError => e + raise ArgumentError, "get bitcoin raw transaction #{txid} failed: #{e}" + end + def rpc @rpc ||= Bitcoin::Rpc.instance end diff --git a/app/models/bitcoin_address_mapping.rb b/app/models/bitcoin_address_mapping.rb index 65dd97e11..36bf95fc4 100644 --- a/app/models/bitcoin_address_mapping.rb +++ b/app/models/bitcoin_address_mapping.rb @@ -15,5 +15,5 @@ class BitcoinAddressMapping < ApplicationRecord # # Indexes # -# idex_bitcon_addresses_on_mapping (bitcoin_address_id,ckb_address_id) UNIQUE +# index_bitcoin_addresses_on_mapping (bitcoin_address_id,ckb_address_id) UNIQUE # diff --git a/app/models/bitcoin_transaction.rb b/app/models/bitcoin_transaction.rb index fa829f577..04cc65b87 100644 --- a/app/models/bitcoin_transaction.rb +++ b/app/models/bitcoin_transaction.rb @@ -1,5 +1,6 @@ class BitcoinTransaction < ApplicationRecord has_many :bitcoin_vouts + has_many :bitcoin_transfers def confirmations tip_block_height = diff --git a/app/models/bitcoin_transfer.rb b/app/models/bitcoin_transfer.rb new file mode 100644 index 000000000..bc447e778 --- /dev/null +++ b/app/models/bitcoin_transfer.rb @@ -0,0 +1,26 @@ +class BitcoinTransfer < ApplicationRecord + belongs_to :bitcoin_transaction + belongs_to :ckb_transaction + belongs_to :cell_output + + enum lock_type: { rgbpp: 0, btc_time: 1 } +end + +# == Schema Information +# +# Table name: bitcoin_transfers +# +# id :bigint not null, primary key +# bitcoin_transaction_id :bigint +# ckb_transaction_id :bigint +# cell_output_id :bigint +# lock_type :integer default("rgbpp") +# created_at :datetime not null +# updated_at :datetime not null +# +# Indexes +# +# index_bitcoin_transfers_on_bitcoin_transaction_id (bitcoin_transaction_id) +# index_bitcoin_transfers_on_cell_output_id (cell_output_id) UNIQUE +# index_bitcoin_transfers_on_ckb_transaction_id (ckb_transaction_id) +# diff --git a/app/models/ckb_sync/new_node_data_processor.rb b/app/models/ckb_sync/new_node_data_processor.rb index 64655952a..b62052182 100644 --- a/app/models/ckb_sync/new_node_data_processor.rb +++ b/app/models/ckb_sync/new_node_data_processor.rb @@ -138,7 +138,7 @@ def detect_token_transfer(token_transfer_ckb_tx_ids) end def detect_bitcoin_transactions(local_block) - BitcoinTransactionDetectWorker.perform_async(local_block.id) + BitcoinTransactionDetectWorker.perform_async(local_block.number) end def process_ckb_txs( diff --git a/app/workers/bitcoin_transaction_detect_worker.rb b/app/workers/bitcoin_transaction_detect_worker.rb index d1b155aaf..46ad06c5b 100644 --- a/app/workers/bitcoin_transaction_detect_worker.rb +++ b/app/workers/bitcoin_transaction_detect_worker.rb @@ -2,65 +2,99 @@ class BitcoinTransactionDetectWorker include Sidekiq::Worker sidekiq_options queue: "bitcoin" - def perform(block_id) - block = Block.find_by(id: block_id) + BITCOIN_RPC_BATCH_SIZE = 30 + + attr_accessor :txids, :rgbpp_cell_ids, :btc_time_cell_ids + + def perform(number) + block = Block.find_by(number:) return unless block + @txids = [] # bitcoin txids + @rgbpp_cell_ids = [] # rgbpp cells + @btc_time_cell_ids = [] # btc time cells + ApplicationRecord.transaction do block.ckb_transactions.each do |transaction| - vin_attributes = [] + inputs = transaction.input_cells + outputs = transaction.cell_outputs + (inputs + outputs).each { |cell| collect_rgb_ids(cell) } + end - # import cell_inputs utxo - transaction.cell_inputs.each do |cell| - previous_cell_output = cell.previous_cell_output - next unless previous_cell_output + # batch fetch bitcoin raw transactions + cache_raw_transactions! + # import rgbpp cells + @rgbpp_cell_ids.each { |cell_id| ImportRgbppCellJob.perform_now(cell_id) } + # import btc time cells + @btc_time_cell_ids.each { |cell_id| ImportBtcTimeCellJob.perform_now(cell_id) } + # update tags + update_transaction_tags!(transaction) + end + end + + def collect_rgb_ids(cell_output) + lock_script = cell_output.lock_script + cell_output_id = cell_output.id - lock_script = previous_cell_output.lock_script - next unless CkbUtils.is_rgbpp_lock_cell?(lock_script) + if CkbUtils.is_rgbpp_lock_cell?(lock_script) + txid, _out_index = CkbUtils.parse_rgbpp_args(lock_script.args) + unless BitcoinTransfer.includes(:bitcoin_transaction).where( + bitcoin_transactions: { txid: }, + bitcoin_transfers: { cell_output_id:, lock_type: "rgbpp" }, + ).exists? + @rgbpp_cell_ids << cell_output_id + @txids << txid + end + end - # import previous bitcoin transaction if prev vout is missing - import_utxo!(lock_script.args, previous_cell_output.id) + if CkbUtils.is_btc_time_lock_cell?(lock_script) + parsed_args = CkbUtils.parse_btc_time_lock_cell(lock_script.args) + txid = parsed_args.txid + unless BitcoinTransfer.includes(:bitcoin_transaction).where( + bitcoin_transactions: { txid: }, + bitcoin_transfers: { cell_output_id:, lock_type: "btc_time" }, + ).exists? + @btc_time_cell_ids << cell_output_id + @txids << txid + end + end + end - previous_vout = BitcoinVout.find_by(cell_output_id: previous_cell_output.id) - vin_attributes << { - previous_bitcoin_vout_id: previous_vout.id, - ckb_transaction_id: transaction.id, - cell_input_id: cell.id, - } - end + def cache_raw_transactions! + return if @txids.empty? - if vin_attributes.present? - BitcoinVin.upsert_all(vin_attributes, unique_by: %i[ckb_transaction_id cell_input_id]) - end + get_raw_transactions = ->(txids) do + payload = txids.map.with_index do |txid, index| + { jsonrpc: "1.0", id: index + 1, method: "getrawtransaction", params: [txid, 2] } + end + response = HTTP.timeout(10).post(ENV["BITCOIN_NODE_URL"], json: payload) + JSON.parse(response.to_s) + end - # import cell_outputs utxo - transaction.cell_outputs.each do |cell| - lock_script = cell.lock_script - next unless CkbUtils.is_rgbpp_lock_cell?(lock_script) + to_cache = {} + not_cached = @txids.uniq.reject { Rails.cache.exist?(_1) } - import_utxo!(lock_script.args, cell.id) - end + not_cached.each_slice(BITCOIN_RPC_BATCH_SIZE).each do |txids| + get_raw_transactions.call(txids).each do |data| + next if data && data["error"].present? - # update transaction rgbpp tags - update_rgbpp_tags!(transaction) + txid = data.dig("result", "txid") + to_cache[txid] = data end end + + Rails.cache.write_multi(to_cache, expires_in: 10.minutes) if to_cache.present? + rescue StandardError => e + Rails.logger.error "cache raw transactions(#{@txids.uniq}) failed: #{e.message}" end - def import_utxo!(args, cell_id) - txid, out_index = CkbUtils.parse_rgbpp_args(args) + def update_transaction_tags!(transaction) + transaction.tags ||= [] - unless BitcoinTransaction.includes(:bitcoin_vouts).where( - bitcoin_transactions: { txid: }, - bitcoin_vouts: { index: out_index, cell_output_id: cell_id }, - ).exists? - ImportBitcoinUtxoJob.perform_now(cell_id) - end - end + cell_output_ids = transaction.input_cell_ids + transaction.cell_output_ids + lock_types = BitcoinTransfer.where(cell_output_id: cell_output_ids).pluck(:lock_type) + transaction.tags += lock_types.compact.uniq - def update_rgbpp_tags!(transaction) - if transaction.bitcoin_vins.exists? || transaction.bitcoin_vouts.exists? - transaction.update!(tags: transaction.tags.to_a + ["rgbpp"]) - end + transaction.update!(tags: transaction.tags.uniq) end end diff --git a/db/migrate/20240428085020_rename_index_of_bitcoin_address_mappings.rb b/db/migrate/20240428085020_rename_index_of_bitcoin_address_mappings.rb new file mode 100644 index 000000000..3800d6e64 --- /dev/null +++ b/db/migrate/20240428085020_rename_index_of_bitcoin_address_mappings.rb @@ -0,0 +1,5 @@ +class RenameIndexOfBitcoinAddressMappings < ActiveRecord::Migration[7.0] + def change + rename_index :bitcoin_address_mappings, "idex_bitcon_addresses_on_mapping", "index_bitcoin_addresses_on_mapping" + end +end diff --git a/db/migrate/20240429102325_create_bitcoin_transfers.rb b/db/migrate/20240429102325_create_bitcoin_transfers.rb new file mode 100644 index 000000000..0e4ce7c89 --- /dev/null +++ b/db/migrate/20240429102325_create_bitcoin_transfers.rb @@ -0,0 +1,16 @@ +class CreateBitcoinTransfers < ActiveRecord::Migration[7.0] + def change + create_table :bitcoin_transfers do |t| + t.bigint :bitcoin_transaction_id + t.bigint :ckb_transaction_id + t.bigint :cell_output_id + t.integer :lock_type, default: 0 + + t.timestamps + end + + add_index :bitcoin_transfers, :ckb_transaction_id + add_index :bitcoin_transfers, :bitcoin_transaction_id + add_index :bitcoin_transfers, :cell_output_id, unique: true + end +end diff --git a/db/structure.sql b/db/structure.sql index 7d66491ad..521cd05ef 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -624,6 +624,39 @@ CREATE SEQUENCE public.bitcoin_statistics_id_seq ALTER SEQUENCE public.bitcoin_statistics_id_seq OWNED BY public.bitcoin_statistics.id; +-- +-- Name: bitcoin_time_locks; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.bitcoin_time_locks ( + id bigint NOT NULL, + bitcoin_transaction_id bigint, + ckb_transaction_id bigint, + cell_output_id bigint, + created_at timestamp(6) without time zone NOT NULL, + updated_at timestamp(6) without time zone NOT NULL +); + + +-- +-- Name: bitcoin_time_locks_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.bitcoin_time_locks_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: bitcoin_time_locks_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.bitcoin_time_locks_id_seq OWNED BY public.bitcoin_time_locks.id; + + -- -- Name: bitcoin_transactions; Type: TABLE; Schema: public; Owner: - -- @@ -659,6 +692,40 @@ CREATE SEQUENCE public.bitcoin_transactions_id_seq ALTER SEQUENCE public.bitcoin_transactions_id_seq OWNED BY public.bitcoin_transactions.id; +-- +-- Name: bitcoin_transfers; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.bitcoin_transfers ( + id bigint NOT NULL, + bitcoin_transaction_id bigint, + ckb_transaction_id bigint, + cell_output_id bigint, + lock_type integer DEFAULT 0, + created_at timestamp(6) without time zone NOT NULL, + updated_at timestamp(6) without time zone NOT NULL +); + + +-- +-- Name: bitcoin_transfers_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.bitcoin_transfers_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: bitcoin_transfers_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.bitcoin_transfers_id_seq OWNED BY public.bitcoin_transfers.id; + + -- -- Name: bitcoin_vins; Type: TABLE; Schema: public; Owner: - -- @@ -2636,6 +2703,13 @@ ALTER TABLE ONLY public.bitcoin_addresses ALTER COLUMN id SET DEFAULT nextval('p ALTER TABLE ONLY public.bitcoin_statistics ALTER COLUMN id SET DEFAULT nextval('public.bitcoin_statistics_id_seq'::regclass); +-- +-- Name: bitcoin_time_locks id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.bitcoin_time_locks ALTER COLUMN id SET DEFAULT nextval('public.bitcoin_time_locks_id_seq'::regclass); + + -- -- Name: bitcoin_transactions id; Type: DEFAULT; Schema: public; Owner: - -- @@ -2643,6 +2717,13 @@ ALTER TABLE ONLY public.bitcoin_statistics ALTER COLUMN id SET DEFAULT nextval(' ALTER TABLE ONLY public.bitcoin_transactions ALTER COLUMN id SET DEFAULT nextval('public.bitcoin_transactions_id_seq'::regclass); +-- +-- Name: bitcoin_transfers id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.bitcoin_transfers ALTER COLUMN id SET DEFAULT nextval('public.bitcoin_transfers_id_seq'::regclass); + + -- -- Name: bitcoin_vins id; Type: DEFAULT; Schema: public; Owner: - -- @@ -3021,6 +3102,14 @@ ALTER TABLE ONLY public.bitcoin_statistics ADD CONSTRAINT bitcoin_statistics_pkey PRIMARY KEY (id); +-- +-- Name: bitcoin_time_locks bitcoin_time_locks_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.bitcoin_time_locks + ADD CONSTRAINT bitcoin_time_locks_pkey PRIMARY KEY (id); + + -- -- Name: bitcoin_transactions bitcoin_transactions_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -3029,6 +3118,14 @@ ALTER TABLE ONLY public.bitcoin_transactions ADD CONSTRAINT bitcoin_transactions_pkey PRIMARY KEY (id); +-- +-- Name: bitcoin_transfers bitcoin_transfers_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.bitcoin_transfers + ADD CONSTRAINT bitcoin_transfers_pkey PRIMARY KEY (id); + + -- -- Name: bitcoin_vins bitcoin_vins_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -3676,13 +3773,6 @@ CREATE INDEX ckb_transactions_rejected_tags_idx ON public.ckb_transactions_rejec CREATE INDEX ckb_transactions_rejected_tx_hash_idx ON public.ckb_transactions_rejected USING hash (tx_hash); --- --- Name: idex_bitcon_addresses_on_mapping; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX idex_bitcon_addresses_on_mapping ON public.bitcoin_address_mappings USING btree (bitcoin_address_id, ckb_address_id); - - -- -- Name: idx_cell_inputs_on_block_id; Type: INDEX; Schema: public; Owner: - -- @@ -3781,6 +3871,13 @@ CREATE INDEX index_addresses_on_lock_hash ON public.addresses USING hash (lock_h CREATE UNIQUE INDEX index_average_block_time_by_hour_on_hour ON public.average_block_time_by_hour USING btree (hour); +-- +-- Name: index_bitcoin_addresses_on_mapping; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX index_bitcoin_addresses_on_mapping ON public.bitcoin_address_mappings USING btree (bitcoin_address_id, ckb_address_id); + + -- -- Name: index_bitcoin_statistics_on_timestamp; Type: INDEX; Schema: public; Owner: - -- @@ -3788,6 +3885,13 @@ CREATE UNIQUE INDEX index_average_block_time_by_hour_on_hour ON public.average_b CREATE UNIQUE INDEX index_bitcoin_statistics_on_timestamp ON public.bitcoin_statistics USING btree ("timestamp"); +-- +-- Name: index_bitcoin_time_locks_on_cell; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX index_bitcoin_time_locks_on_cell ON public.bitcoin_time_locks USING btree (bitcoin_transaction_id, cell_output_id); + + -- -- Name: index_bitcoin_transactions_on_txid; Type: INDEX; Schema: public; Owner: - -- @@ -3795,6 +3899,27 @@ CREATE UNIQUE INDEX index_bitcoin_statistics_on_timestamp ON public.bitcoin_stat CREATE UNIQUE INDEX index_bitcoin_transactions_on_txid ON public.bitcoin_transactions USING btree (txid); +-- +-- Name: index_bitcoin_transfers_on_bitcoin_transaction_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_bitcoin_transfers_on_bitcoin_transaction_id ON public.bitcoin_transfers USING btree (bitcoin_transaction_id); + + +-- +-- Name: index_bitcoin_transfers_on_cell_output_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX index_bitcoin_transfers_on_cell_output_id ON public.bitcoin_transfers USING btree (cell_output_id); + + +-- +-- Name: index_bitcoin_transfers_on_ckb_transaction_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_bitcoin_transfers_on_ckb_transaction_id ON public.bitcoin_transfers USING btree (ckb_transaction_id); + + -- -- Name: index_bitcoin_vins_on_ckb_transaction_id; Type: INDEX; Schema: public; Owner: - -- @@ -5030,6 +5155,8 @@ INSERT INTO "schema_migrations" (version) VALUES ('20240408065818'), ('20240408075718'), ('20240408082159'), -('20240415080556'); +('20240415080556'), +('20240428085020'), +('20240429102325'); diff --git a/lib/tasks/migration/generate_bitcoin_transfers.rake b/lib/tasks/migration/generate_bitcoin_transfers.rake new file mode 100644 index 000000000..c36cd5580 --- /dev/null +++ b/lib/tasks/migration/generate_bitcoin_transfers.rake @@ -0,0 +1,34 @@ +namespace :migration do + desc "Usage: RAILS_ENV=production bundle exec rake migration:generate_bitcoin_transfers" + task generate_bitcoin_transfers: :environment do + block_numbers = [] + binary_hashes = CkbUtils.hexes_to_bins_sql( + [CkbSync::Api.instance.rgbpp_code_hash, CkbSync::Api.instance.btc_time_code_hash], + ) + + LockScript.where("code_hash IN (#{binary_hashes})").find_in_batches(batch_size: 50) do |lock_scripts| + CellOutput.where(lock_script_id: lock_scripts.map(&:id)).find_each do |cell_output| + ckb_transaction = cell_output.ckb_transaction + # next if BitcoinTransfer.exists?(ckb_transaction:, cell_output:) + + block_number = ckb_transaction.block_number + next if block_numbers.include?(block_number) + + block_numbers << block_number + end + end + + progress_bar = ProgressBar.create( + { + total: block_numbers.length, + format: "%e %B %p%% %c/%C", + }, + ) + block_numbers.sort.each do |block_number| + BitcoinTransactionDetectWorker.new.perform(block_number) + progress_bar.increment + end + + puts "done" + end +end diff --git a/test/factories/bitcoin_transfers.rb b/test/factories/bitcoin_transfers.rb new file mode 100644 index 000000000..17935eeed --- /dev/null +++ b/test/factories/bitcoin_transfers.rb @@ -0,0 +1,5 @@ +FactoryBot.define do + factory :bitcoin_transfer do + + end +end diff --git a/test/models/bitcoin_transfer_test.rb b/test/models/bitcoin_transfer_test.rb new file mode 100644 index 000000000..9e16b6f53 --- /dev/null +++ b/test/models/bitcoin_transfer_test.rb @@ -0,0 +1,7 @@ +require "test_helper" + +class BitcoinTransferTest < ActiveSupport::TestCase + # test "the truth" do + # assert true + # end +end From 438be672fef6934598d9a7b153b56d29ecc15819 Mon Sep 17 00:00:00 2001 From: Rabbit Date: Tue, 30 Apr 2024 13:53:48 +0800 Subject: [PATCH 2/8] chore: catch generate bitcoin transfers error (#1827) --- app/jobs/import_btc_time_cell_job.rb | 6 ++---- app/jobs/import_rgbpp_cell_job.rb | 6 ++---- lib/tasks/migration/generate_bitcoin_transfers.rake | 7 ++++++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/app/jobs/import_btc_time_cell_job.rb b/app/jobs/import_btc_time_cell_job.rb index f22ac5d95..4150cb08a 100644 --- a/app/jobs/import_btc_time_cell_job.rb +++ b/app/jobs/import_btc_time_cell_job.rb @@ -44,13 +44,11 @@ def build_transaction!(raw_tx) def fetch_raw_transaction(txid) data = Rails.cache.read(txid) data ||= rpc.getrawtransaction(txid, 2) - - return if data && data["error"].present? - Rails.cache.write(txid, data, expires_in: 10.minutes) unless Rails.cache.exist?(txid) data rescue StandardError => e - raise ArgumentError, "get bitcoin raw transaction #{txid} failed: #{e}" + Rails.logger.error "get bitcoin raw transaction #{txid} failed: #{e}" + nil end def rpc diff --git a/app/jobs/import_rgbpp_cell_job.rb b/app/jobs/import_rgbpp_cell_job.rb index 2c522e421..2f3cf519d 100644 --- a/app/jobs/import_rgbpp_cell_job.rb +++ b/app/jobs/import_rgbpp_cell_job.rb @@ -137,13 +137,11 @@ def build_address!(address_hash, cell_output) def fetch_raw_transaction(txid) data = Rails.cache.read(txid) data ||= rpc.getrawtransaction(txid, 2) - - return if data && data["error"].present? - Rails.cache.write(txid, data, expires_in: 10.minutes) unless Rails.cache.exist?(txid) data rescue StandardError => e - raise ArgumentError, "get bitcoin raw transaction #{txid} failed: #{e}" + Rails.logger.error "get bitcoin raw transaction #{txid} failed: #{e}" + nil end def rpc diff --git a/lib/tasks/migration/generate_bitcoin_transfers.rake b/lib/tasks/migration/generate_bitcoin_transfers.rake index c36cd5580..802ddce39 100644 --- a/lib/tasks/migration/generate_bitcoin_transfers.rake +++ b/lib/tasks/migration/generate_bitcoin_transfers.rake @@ -25,7 +25,12 @@ namespace :migration do }, ) block_numbers.sort.each do |block_number| - BitcoinTransactionDetectWorker.new.perform(block_number) + begin + BitcoinTransactionDetectWorker.new.perform(block_number) + rescue StandardError => e + Rails.logger.error "Failed to process block number #{block_number}: #{e}" + end + progress_bar.increment end From 0e026372e0f13e9a197d17fdbcc15b3d69f0cb7f Mon Sep 17 00:00:00 2001 From: Rabbit Date: Tue, 30 Apr 2024 16:38:54 +0800 Subject: [PATCH 3/8] fix: bitcoin raw transaction (#1828) * fix: bitcoin raw transaction * chore: check if btc time transaciton * chore: update bitcoin transaction confirmations * fix: rgb digest --- .../api/v2/ckb_transactions_controller.rb | 16 ++++++----- app/jobs/import_btc_time_cell_job.rb | 5 ++-- app/jobs/import_rgbpp_cell_job.rb | 4 +-- app/models/bitcoin_transaction.rb | 5 ++-- .../concerns/ckb_transactions/bitcoin.rb | 12 +------- app/services/bitcoin/rpc.rb | 2 +- .../bitcoin_transaction_detect_worker.rb | 28 ++++++++++--------- .../migration/generate_bitcoin_transfers.rake | 10 +++---- 8 files changed, 37 insertions(+), 45 deletions(-) diff --git a/app/controllers/api/v2/ckb_transactions_controller.rb b/app/controllers/api/v2/ckb_transactions_controller.rb index 0d5145a1c..503f7c599 100644 --- a/app/controllers/api/v2/ckb_transactions_controller.rb +++ b/app/controllers/api/v2/ckb_transactions_controller.rb @@ -63,16 +63,18 @@ def rgb_digest end end + bitcoin_transaction = BitcoinTransaction.includes(:bitcoin_vouts).find_by( + bitcoin_vouts: { ckb_transaction_id: @ckb_transaction.id }, + ) op_return = @ckb_transaction.bitcoin_vouts.find_by(op_return: true) - bitcoin_transaction = BitcoinTransaction.includes(:bitcoin_vouts).find_by(bitcoin_vouts: { ckb_transaction_id: @ckb_transaction.id }) + if op_return && bitcoin_transaction + txid = bitcoin_transaction.txid + commitment = op_return.commitment + confirmations = bitcoin_transaction.confirmations + end render json: { - data: { - txid: bitcoin_transaction&.txid, - confirmations: bitcoin_transaction&.confirmations, - commitment: op_return&.commitment, - transfers:, - }, + data: { txid:, confirmations:, commitment:, transfers: }, } end diff --git a/app/jobs/import_btc_time_cell_job.rb b/app/jobs/import_btc_time_cell_job.rb index 4150cb08a..5025b8875 100644 --- a/app/jobs/import_btc_time_cell_job.rb +++ b/app/jobs/import_btc_time_cell_job.rb @@ -10,7 +10,8 @@ def perform(cell_id) return unless CkbUtils.is_btc_time_lock_cell?(lock_script) parsed_args = CkbUtils.parse_btc_time_lock_cell(lock_script.args) - Rails.logger.info("Importing btc time cell txid #{parsed_args.txid}") + txid = parsed_args.txid + Rails.logger.info("Importing btc time cell #{cell_id} txid #{txid}") # build bitcoin transaction raw_tx = fetch_raw_transaction(txid) @@ -45,7 +46,7 @@ def fetch_raw_transaction(txid) data = Rails.cache.read(txid) data ||= rpc.getrawtransaction(txid, 2) Rails.cache.write(txid, data, expires_in: 10.minutes) unless Rails.cache.exist?(txid) - data + data["result"] rescue StandardError => e Rails.logger.error "get bitcoin raw transaction #{txid} failed: #{e}" nil diff --git a/app/jobs/import_rgbpp_cell_job.rb b/app/jobs/import_rgbpp_cell_job.rb index 2f3cf519d..5b8a6064e 100644 --- a/app/jobs/import_rgbpp_cell_job.rb +++ b/app/jobs/import_rgbpp_cell_job.rb @@ -10,7 +10,7 @@ def perform(cell_id) return unless CkbUtils.is_rgbpp_lock_cell?(lock_script) txid, out_index = CkbUtils.parse_rgbpp_args(lock_script.args) - Rails.logger.info("Importing rgbpp cell txid #{txid} out_index #{out_index}") + Rails.logger.info("Importing rgbpp cell #{cell_id} txid #{txid} out_index #{out_index}") # build bitcoin transaction raw_tx = fetch_raw_transaction(txid) @@ -138,7 +138,7 @@ def fetch_raw_transaction(txid) data = Rails.cache.read(txid) data ||= rpc.getrawtransaction(txid, 2) Rails.cache.write(txid, data, expires_in: 10.minutes) unless Rails.cache.exist?(txid) - data + data["result"] rescue StandardError => e Rails.logger.error "get bitcoin raw transaction #{txid} failed: #{e}" nil diff --git a/app/models/bitcoin_transaction.rb b/app/models/bitcoin_transaction.rb index 04cc65b87..1d17dd81d 100644 --- a/app/models/bitcoin_transaction.rb +++ b/app/models/bitcoin_transaction.rb @@ -5,9 +5,8 @@ class BitcoinTransaction < ApplicationRecord def confirmations tip_block_height = Rails.cache.fetch("tip_block_height", expires_in: 5.minutes) do - blocks = Bitcoin::Rpc.instance.getchaintips - tip_block = blocks.find { |h| h["status"] == "active" } - tip_block["height"] + chain_info = Bitcoin::Rpc.instance.getblockchaininfo + chain_info["headers"] rescue StandardError => e Rails.logger.error "get tip block faild: #{e.message}" nil diff --git a/app/models/concerns/ckb_transactions/bitcoin.rb b/app/models/concerns/ckb_transactions/bitcoin.rb index d2c173f1d..399060400 100644 --- a/app/models/concerns/ckb_transactions/bitcoin.rb +++ b/app/models/concerns/ckb_transactions/bitcoin.rb @@ -12,17 +12,7 @@ def rgb_transaction? end def btc_time_transaction? - is_btc_time_lock_cell = ->(lock_script) { CkbUtils.is_btc_time_lock_cell?(lock_script) } - inputs.includes(:lock_script).any? { is_btc_time_lock_cell.call(_1.lock_script) } || - outputs.includes(:lock_script).any? { is_btc_time_lock_cell.call(_1.lock_script) } - end - - def rgb_commitment - return unless rgb_transaction? - - # In the outputs, there is exactly one OP_RETURN containing a commitment. - op_return = bitcoin_vouts.find_by(op_return: true) - op_return&.commitment + !!tags&.include?("btc_time") end def rgb_txid diff --git a/app/services/bitcoin/rpc.rb b/app/services/bitcoin/rpc.rb index d2644026d..fdd16ed95 100644 --- a/app/services/bitcoin/rpc.rb +++ b/app/services/bitcoin/rpc.rb @@ -2,7 +2,7 @@ module Bitcoin class Rpc include Singleton - METHOD_NAMES = %w(getchaintips getrawtransaction getblock getblockhash getblockheader) + METHOD_NAMES = %w(getchaintips getrawtransaction getblock getblockhash getblockheader getblockchaininfo) def initialize(endpoint = ENV["BITCOIN_NODE_URL"]) @endpoint = endpoint @id = 0 diff --git a/app/workers/bitcoin_transaction_detect_worker.rb b/app/workers/bitcoin_transaction_detect_worker.rb index 46ad06c5b..2ba799e73 100644 --- a/app/workers/bitcoin_transaction_detect_worker.rb +++ b/app/workers/bitcoin_transaction_detect_worker.rb @@ -4,11 +4,11 @@ class BitcoinTransactionDetectWorker BITCOIN_RPC_BATCH_SIZE = 30 - attr_accessor :txids, :rgbpp_cell_ids, :btc_time_cell_ids + attr_accessor :block, :txids, :rgbpp_cell_ids, :btc_time_cell_ids def perform(number) - block = Block.find_by(number:) - return unless block + @block = Block.find_by(number:) + return unless @block @txids = [] # bitcoin txids @rgbpp_cell_ids = [] # rgbpp cells @@ -18,17 +18,17 @@ def perform(number) block.ckb_transactions.each do |transaction| inputs = transaction.input_cells outputs = transaction.cell_outputs - (inputs + outputs).each { |cell| collect_rgb_ids(cell) } + (inputs + outputs).each { collect_rgb_ids(_1) } end # batch fetch bitcoin raw transactions cache_raw_transactions! # import rgbpp cells - @rgbpp_cell_ids.each { |cell_id| ImportRgbppCellJob.perform_now(cell_id) } + @rgbpp_cell_ids.each { ImportRgbppCellJob.perform_now(_1) } # import btc time cells - @btc_time_cell_ids.each { |cell_id| ImportBtcTimeCellJob.perform_now(cell_id) } + @btc_time_cell_ids.each { ImportBtcTimeCellJob.perform_now(_1) } # update tags - update_transaction_tags!(transaction) + update_transaction_tags! end end @@ -88,13 +88,15 @@ def cache_raw_transactions! Rails.logger.error "cache raw transactions(#{@txids.uniq}) failed: #{e.message}" end - def update_transaction_tags!(transaction) - transaction.tags ||= [] + def update_transaction_tags! + @block.ckb_transactions.each do |transaction| + transaction.tags ||= [] - cell_output_ids = transaction.input_cell_ids + transaction.cell_output_ids - lock_types = BitcoinTransfer.where(cell_output_id: cell_output_ids).pluck(:lock_type) - transaction.tags += lock_types.compact.uniq + cell_output_ids = transaction.input_cell_ids + transaction.cell_output_ids + lock_types = BitcoinTransfer.where(cell_output_id: cell_output_ids).pluck(:lock_type) + transaction.tags += lock_types.compact.uniq - transaction.update!(tags: transaction.tags.uniq) + transaction.update!(tags: transaction.tags.uniq) + end end end diff --git a/lib/tasks/migration/generate_bitcoin_transfers.rake b/lib/tasks/migration/generate_bitcoin_transfers.rake index 802ddce39..e857abbbf 100644 --- a/lib/tasks/migration/generate_bitcoin_transfers.rake +++ b/lib/tasks/migration/generate_bitcoin_transfers.rake @@ -9,7 +9,7 @@ namespace :migration do LockScript.where("code_hash IN (#{binary_hashes})").find_in_batches(batch_size: 50) do |lock_scripts| CellOutput.where(lock_script_id: lock_scripts.map(&:id)).find_each do |cell_output| ckb_transaction = cell_output.ckb_transaction - # next if BitcoinTransfer.exists?(ckb_transaction:, cell_output:) + next if BitcoinTransfer.exists?(ckb_transaction:, cell_output:) block_number = ckb_transaction.block_number next if block_numbers.include?(block_number) @@ -25,11 +25,9 @@ namespace :migration do }, ) block_numbers.sort.each do |block_number| - begin - BitcoinTransactionDetectWorker.new.perform(block_number) - rescue StandardError => e - Rails.logger.error "Failed to process block number #{block_number}: #{e}" - end + BitcoinTransactionDetectWorker.new.perform(block_number) + rescue StandardError => e + Rails.logger.error "Failed to process block number #{block_number}: #{e}" progress_bar.increment end From b755f0a6add452555631bf3f3e0da80210adeba7 Mon Sep 17 00:00:00 2001 From: Miles Zhang Date: Mon, 6 May 2024 18:59:42 +0800 Subject: [PATCH 4/8] feat: use task to check cell output capacity (#1829) Signed-off-by: Miles Zhang --- .../migration/check_output_capacity.rake | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 lib/tasks/migration/check_output_capacity.rake diff --git a/lib/tasks/migration/check_output_capacity.rake b/lib/tasks/migration/check_output_capacity.rake new file mode 100644 index 000000000..68e5e4daa --- /dev/null +++ b/lib/tasks/migration/check_output_capacity.rake @@ -0,0 +1,38 @@ +namespace :migration do + desc "Usage: RAILS_ENV=production bundle exec rake migration:check_output_capacity[0, 10000]" + task :check_output_capacity, %i[start_block end_block] => :environment do |_, args| + (args[:start_block].to_i..args[:end_block].to_i).to_a.each do |number| + check_capacity(number, 0) + end; nil + + puts "done" + + def check_capacity(number, retry_count) + target_block = CkbSync::Api.instance.get_block_by_number(number) + txs = target_block.transactions + txs.each do |tx| + tx.inputs.each do |input| + unless input.previous_output.tx_hash == "0x0000000000000000000000000000000000000000000000000000000000000000" + result = CellOutput.where(tx_hash: input.previous_output.tx_hash, cell_index: input.previous_output.index, status: :dead).exists? + unless result + puts "#{input.previous_output.tx_hash}-#{input.index}" + end + end + end + tx.outputs.each_with_index do |output, index| + db_output = CellOutput.find_by(tx_hash: tx.hash, cell_index: index) + if db_output.capacity != output.capacity + puts "#{tx.hash}-#{index}" + end + end + end; nil + rescue StandardError => _e + retry_count += 1 + if retry_count > 2 + puts number + else + check_capacity(number, retry_count) + end + end + end +end From 7b2cdee6bac26f2ac4d70f8ac64545aec4f1ee3c Mon Sep 17 00:00:00 2001 From: Miles Zhang Date: Mon, 6 May 2024 19:19:26 +0800 Subject: [PATCH 5/8] fix: move method out of task (#1830) Signed-off-by: Miles Zhang --- .../migration/check_output_capacity.rake | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/lib/tasks/migration/check_output_capacity.rake b/lib/tasks/migration/check_output_capacity.rake index 68e5e4daa..6d502500a 100644 --- a/lib/tasks/migration/check_output_capacity.rake +++ b/lib/tasks/migration/check_output_capacity.rake @@ -6,33 +6,33 @@ namespace :migration do end; nil puts "done" + end - def check_capacity(number, retry_count) - target_block = CkbSync::Api.instance.get_block_by_number(number) - txs = target_block.transactions - txs.each do |tx| - tx.inputs.each do |input| - unless input.previous_output.tx_hash == "0x0000000000000000000000000000000000000000000000000000000000000000" - result = CellOutput.where(tx_hash: input.previous_output.tx_hash, cell_index: input.previous_output.index, status: :dead).exists? - unless result - puts "#{input.previous_output.tx_hash}-#{input.index}" - end + def check_capacity(number, retry_count) + target_block = CkbSync::Api.instance.get_block_by_number(number) + txs = target_block.transactions + txs.each do |tx| + tx.inputs.each do |input| + unless input.previous_output.tx_hash == "0x0000000000000000000000000000000000000000000000000000000000000000" + result = CellOutput.where(tx_hash: input.previous_output.tx_hash, cell_index: input.previous_output.index, status: :dead).exists? + unless result + puts "#{input.previous_output.tx_hash}-#{input.index}" end end - tx.outputs.each_with_index do |output, index| - db_output = CellOutput.find_by(tx_hash: tx.hash, cell_index: index) - if db_output.capacity != output.capacity - puts "#{tx.hash}-#{index}" - end + end + tx.outputs.each_with_index do |output, index| + db_output = CellOutput.find_by(tx_hash: tx.hash, cell_index: index) + if db_output.capacity != output.capacity + puts "#{tx.hash}-#{index}" end - end; nil - rescue StandardError => _e - retry_count += 1 - if retry_count > 2 - puts number - else - check_capacity(number, retry_count) end + end; nil + rescue StandardError => _e + retry_count += 1 + if retry_count > 2 + puts number + else + check_capacity(number, retry_count) end end end From cb3d708a07bbe42c91f9b54ec1454a3283ed9415 Mon Sep 17 00:00:00 2001 From: Miles Zhang Date: Tue, 7 May 2024 14:28:09 +0800 Subject: [PATCH 6/8] feat: generate h24_ckb_transactions_count of contract (#1831) * feat: generate h24_ckb_transactions_count of contract Signed-off-by: Miles Zhang * fix: stale? needs to pass active record Signed-off-by: Miles Zhang --------- Signed-off-by: Miles Zhang --- .../api/v2/statistics_controller.rb | 41 +++++++------ app/models/ckb_transaction.rb | 1 + app/models/contract.rb | 1 + app/workers/contract_statistic_worker.rb | 2 + ..._h24_ckb_transactions_count_to_contract.rb | 5 ++ db/structure.sql | 61 ++----------------- 6 files changed, 36 insertions(+), 75 deletions(-) create mode 100644 db/migrate/20240507041552_add_h24_ckb_transactions_count_to_contract.rb diff --git a/app/controllers/api/v2/statistics_controller.rb b/app/controllers/api/v2/statistics_controller.rb index bf18d73f6..60e2bfefe 100644 --- a/app/controllers/api/v2/statistics_controller.rb +++ b/app/controllers/api/v2/statistics_controller.rb @@ -1,31 +1,36 @@ module Api::V2 class StatisticsController < BaseController def transaction_fees - expires_in 15.seconds, public: true stats_info = StatisticInfo.default + if stale?(stats_info) + expires_in 15.seconds, public: true - render json: { - transaction_fee_rates: stats_info.transaction_fee_rates, - pending_transaction_fee_rates: stats_info.pending_transaction_fee_rates, - last_n_days_transaction_fee_rates: stats_info.last_n_days_transaction_fee_rates, - } + render json: { + transaction_fee_rates: stats_info.transaction_fee_rates, + pending_transaction_fee_rates: stats_info.pending_transaction_fee_rates, + last_n_days_transaction_fee_rates: stats_info.last_n_days_transaction_fee_rates, + } + end end def contract_resource_distributed - expires_in 30.minutes, public: true + contracts = Contract.filter_nil_hash_type + if stale?(contracts) + expires_in 30.minutes, public: true + json = contracts.map do |contract| + { + name: contract.name, + code_hash: contract.code_hash, + hash_type: contract.hash_type, + tx_count: contract.ckb_transactions_count, + h24_tx_count: contract.h24_ckb_transactions_count, + ckb_amount: (contract.total_referring_cells_capacity / 10**8).truncate(8), + address_count: contract.addresses_count, + } + end - json = Contract.filter_nil_hash_type.map do |contract| - { - name: contract.name, - code_hash: contract.code_hash, - hash_type: contract.hash_type, - tx_count: contract.ckb_transactions_count, - ckb_amount: (contract.total_referring_cells_capacity / 10**8).truncate(8), - address_count: contract.addresses_count, - } + render json: end - - render json: end end end diff --git a/app/models/ckb_transaction.rb b/app/models/ckb_transaction.rb index 81cdd6e9d..5e94c1243 100644 --- a/app/models/ckb_transaction.rb +++ b/app/models/ckb_transaction.rb @@ -52,6 +52,7 @@ class CkbTransaction < ApplicationRecord created_after(start_block_timestamp).created_before(end_block_timestamp) } scope :inner_block, ->(block_id) { where("block_id = ?", block_id) } + scope :h24, -> { where("block_timestamp >= ?", 24.hours.ago.to_i * 1000) } after_commit :flush_cache before_destroy :recover_dead_cell diff --git a/app/models/contract.rb b/app/models/contract.rb index 1db207239..ccd01a011 100644 --- a/app/models/contract.rb +++ b/app/models/contract.rb @@ -44,6 +44,7 @@ def self.create_initial_data # total_deployed_cells_capacity :decimal(30, ) default(0) # total_referring_cells_capacity :decimal(30, ) default(0) # addresses_count :integer +# h24_ckb_transactions_count :integer # # Indexes # diff --git a/app/workers/contract_statistic_worker.rb b/app/workers/contract_statistic_worker.rb index 8ad6968a4..942c22254 100644 --- a/app/workers/contract_statistic_worker.rb +++ b/app/workers/contract_statistic_worker.rb @@ -3,9 +3,11 @@ class ContractStatisticWorker sidekiq_options queue: "critical" def perform + h24_tx_ids = CkbTransaction.h24.pluck(:id) Contract.find_each do |contract| contract.update( ckb_transactions_count: contract.cell_dependencies.count, + h24_ckb_transactions_count: contract.cell_dependencies.where(ckb_transaction_id: h24_tx_ids).count, deployed_cells_count: contract.deployed_cell_outputs&.live&.size, referring_cells_count: contract.referring_cell_outputs&.live&.size, total_deployed_cells_capacity: contract.deployed_cell_outputs&.live&.sum(:capacity), diff --git a/db/migrate/20240507041552_add_h24_ckb_transactions_count_to_contract.rb b/db/migrate/20240507041552_add_h24_ckb_transactions_count_to_contract.rb new file mode 100644 index 000000000..27c46036e --- /dev/null +++ b/db/migrate/20240507041552_add_h24_ckb_transactions_count_to_contract.rb @@ -0,0 +1,5 @@ +class AddH24CkbTransactionsCountToContract < ActiveRecord::Migration[7.0] + def change + add_column :contracts, :h24_ckb_transactions_count, :integer + end +end diff --git a/db/structure.sql b/db/structure.sql index 521cd05ef..10efc727a 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -624,39 +624,6 @@ CREATE SEQUENCE public.bitcoin_statistics_id_seq ALTER SEQUENCE public.bitcoin_statistics_id_seq OWNED BY public.bitcoin_statistics.id; --- --- Name: bitcoin_time_locks; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.bitcoin_time_locks ( - id bigint NOT NULL, - bitcoin_transaction_id bigint, - ckb_transaction_id bigint, - cell_output_id bigint, - created_at timestamp(6) without time zone NOT NULL, - updated_at timestamp(6) without time zone NOT NULL -); - - --- --- Name: bitcoin_time_locks_id_seq; Type: SEQUENCE; Schema: public; Owner: - --- - -CREATE SEQUENCE public.bitcoin_time_locks_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - --- --- Name: bitcoin_time_locks_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - --- - -ALTER SEQUENCE public.bitcoin_time_locks_id_seq OWNED BY public.bitcoin_time_locks.id; - - -- -- Name: bitcoin_transactions; Type: TABLE; Schema: public; Owner: - -- @@ -1302,7 +1269,8 @@ CREATE TABLE public.contracts ( referring_cells_count numeric(30,0) DEFAULT 0.0, total_deployed_cells_capacity numeric(30,0) DEFAULT 0.0, total_referring_cells_capacity numeric(30,0) DEFAULT 0.0, - addresses_count integer + addresses_count integer, + h24_ckb_transactions_count integer ); @@ -2703,13 +2671,6 @@ ALTER TABLE ONLY public.bitcoin_addresses ALTER COLUMN id SET DEFAULT nextval('p ALTER TABLE ONLY public.bitcoin_statistics ALTER COLUMN id SET DEFAULT nextval('public.bitcoin_statistics_id_seq'::regclass); --- --- Name: bitcoin_time_locks id; Type: DEFAULT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.bitcoin_time_locks ALTER COLUMN id SET DEFAULT nextval('public.bitcoin_time_locks_id_seq'::regclass); - - -- -- Name: bitcoin_transactions id; Type: DEFAULT; Schema: public; Owner: - -- @@ -3102,14 +3063,6 @@ ALTER TABLE ONLY public.bitcoin_statistics ADD CONSTRAINT bitcoin_statistics_pkey PRIMARY KEY (id); --- --- Name: bitcoin_time_locks bitcoin_time_locks_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.bitcoin_time_locks - ADD CONSTRAINT bitcoin_time_locks_pkey PRIMARY KEY (id); - - -- -- Name: bitcoin_transactions bitcoin_transactions_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -3885,13 +3838,6 @@ CREATE UNIQUE INDEX index_bitcoin_addresses_on_mapping ON public.bitcoin_address CREATE UNIQUE INDEX index_bitcoin_statistics_on_timestamp ON public.bitcoin_statistics USING btree ("timestamp"); --- --- Name: index_bitcoin_time_locks_on_cell; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX index_bitcoin_time_locks_on_cell ON public.bitcoin_time_locks USING btree (bitcoin_transaction_id, cell_output_id); - - -- -- Name: index_bitcoin_transactions_on_txid; Type: INDEX; Schema: public; Owner: - -- @@ -5157,6 +5103,7 @@ INSERT INTO "schema_migrations" (version) VALUES ('20240408082159'), ('20240415080556'), ('20240428085020'), -('20240429102325'); +('20240429102325'), +('20240507041552'); From fccd768ab96ae631b77c6a93674bbd700d477581 Mon Sep 17 00:00:00 2001 From: Rabbit Date: Tue, 7 May 2024 15:17:56 +0800 Subject: [PATCH 7/8] feat: export xudt snapshot on a specific block (#1832) --- app/controllers/api/v1/xudts_controller.rb | 10 ++++ .../csv_exportable/export_udt_snapshot_job.rb | 58 +++++++++++++++++++ config/routes.rb | 1 + 3 files changed, 69 insertions(+) create mode 100644 app/jobs/csv_exportable/export_udt_snapshot_job.rb diff --git a/app/controllers/api/v1/xudts_controller.rb b/app/controllers/api/v1/xudts_controller.rb index de78d42d4..8d18cd9fa 100644 --- a/app/controllers/api/v1/xudts_controller.rb +++ b/app/controllers/api/v1/xudts_controller.rb @@ -48,6 +48,16 @@ def download_csv raise Api::V1::Exceptions::UdtNotFoundError end + def snapshot + args = params.permit(:id, :number) + file = CsvExportable::ExportUdtSnapshotJob.perform_now(args.to_h) + + send_data file, type: "text/csv; charset=utf-8; header=present", + disposition: "attachment;filename=xudt_snapshot.csv" + rescue ActiveRecord::RecordNotFound + raise Api::V1::Exceptions::UdtNotFoundError + end + private def validate_query_params diff --git a/app/jobs/csv_exportable/export_udt_snapshot_job.rb b/app/jobs/csv_exportable/export_udt_snapshot_job.rb new file mode 100644 index 000000000..e2720432e --- /dev/null +++ b/app/jobs/csv_exportable/export_udt_snapshot_job.rb @@ -0,0 +1,58 @@ +module CsvExportable + class ExportUdtSnapshotJob < BaseExporter + attr_accessor :udt, :block + + def perform(args) + @block = Block.find_by!(number: args[:number]) + @udt = Udt.published_xudt.find_by!(type_hash: args[:id]) + type_script = TypeScript.find_by(@udt.type_script) + + condition = <<-SQL + type_script_id = #{type_script.id} AND + block_timestamp <= #{@block.timestamp} AND + (consumed_block_timestamp > #{@block.timestamp} OR consumed_block_timestamp IS NULL) + SQL + snapshot = CellOutput.where(condition).group(:address_id).sum(:udt_amount) + + rows = [] + snapshot = snapshot.reject { |_, v| v.to_f.zero? } + snapshot.sort_by { |_k, v| -v }.each do |address_id, udt_amount| + row = generate_row(address_id, udt_amount) + next if row.blank? + + rows << row + end + + header = ["Token Symbol", "Block Height", "UnixTimestamp", "date(UTC)", "Address", "Amount"] + generate_csv(header, rows) + end + + def generate_row(address_id, udt_amount) + address = Address.find_by(id: address_id) + return unless address + + address_hash = address.bitcoin_address&.address_hash || address.address_hash + datetime = datetime_utc(@block.timestamp) + + if (decimal = @udt.decimal) + [ + @udt.symbol, + @block.number, + @block.timestamp, + datetime, + address_hash, + parse_udt_amount(udt_amount.to_d, decimal), + ] + else + [ + @udt.symbol, + @block.number, + @block.timestamp, + datetime, + address_hash, + "#{udt_amount} (raw)", + ] + end + end + end +end diff --git a/config/routes.rb b/config/routes.rb index eb90d099a..7b360e95d 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -62,6 +62,7 @@ resources :xudts, only: %i(index show) do collection do get :download_csv + get :snapshot end end resources :omiga_inscriptions, only: %i(index show) do From 49f41cad2ef0709c458eb315f8cce5672fbcc236 Mon Sep 17 00:00:00 2001 From: Miles Zhang Date: Tue, 7 May 2024 19:22:25 +0800 Subject: [PATCH 8/8] feat: always parse nrc 721 cell info (#1833) Signed-off-by: Miles Zhang --- .../ckb_sync/new_node_data_processor.rb | 31 ++++++++----------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/app/models/ckb_sync/new_node_data_processor.rb b/app/models/ckb_sync/new_node_data_processor.rb index b62052182..8c19d98bd 100644 --- a/app/models/ckb_sync/new_node_data_processor.rb +++ b/app/models/ckb_sync/new_node_data_processor.rb @@ -667,7 +667,8 @@ def build_udts!(local_block, outputs, outputs_data) nft_token_attr = { full_name: nil, icon_file: nil, published: false, symbol: nil, decimal: nil, nrc_factory_cell_id: nil } issuer_address = CkbUtils.generate_address(output.lock, CKB::Address::Version::CKB2021) - if cell_type == "m_nft_token" + case cell_type + when "m_nft_token" m_nft_class_type = TypeScript.where(code_hash: CkbSync::Api.instance.token_class_script_code_hash, args: output.type.args[0..49]).first if m_nft_class_type.present? @@ -685,8 +686,7 @@ def build_udts!(local_block, outputs, outputs_data) nft_token_attr[:icon_file] = parsed_class_data.renderer nft_token_attr[:published] = true end - end - if cell_type == "spore_cell" + when "spore_cell" nft_token_attr[:published] = true parsed_spore_cell = CkbUtils.parse_spore_cell_data(outputs_data[tx_index][index]) if parsed_spore_cell[:cluster_id].present? @@ -698,31 +698,26 @@ def build_udts!(local_block, outputs, outputs_data) parsed_cluster_data = CkbUtils.parse_spore_cluster_data(spore_cluster_cell.data) nft_token_attr[:full_name] = parsed_cluster_data[:name] end - end - if cell_type == "nrc_721_token" + when "nrc_721_token" factory_cell = CkbUtils.parse_nrc_721_args(output.type.args) nrc_721_factory_cell = NrcFactoryCell.create_or_find_by(code_hash: factory_cell.code_hash, hash_type: factory_cell.hash_type, args: factory_cell.args) - if nrc_721_factory_cell.verified - nft_token_attr[:full_name] = nrc_721_factory_cell.name - nft_token_attr[:symbol] = - nrc_721_factory_cell.symbol.to_s[0, 16] - nft_token_attr[:icon_file] = - "#{nrc_721_factory_cell.base_token_uri}/#{factory_cell.token_id}" - # refactor: remove this attribute then add udt_id to NrcFactoryCell - nft_token_attr[:nrc_factory_cell_id] = nrc_721_factory_cell.id - end + nft_token_attr[:full_name] = nrc_721_factory_cell.name + nft_token_attr[:symbol] = + nrc_721_factory_cell.symbol.to_s[0, 16] + nft_token_attr[:icon_file] = + "#{nrc_721_factory_cell.base_token_uri}/#{factory_cell.token_id}" + # refactor: remove this attribute then add udt_id to NrcFactoryCell + nft_token_attr[:nrc_factory_cell_id] = nrc_721_factory_cell.id nft_token_attr[:published] = true - end - if cell_type == "omiga_inscription_info" + when "omiga_inscription_info" info = CkbUtils.parse_omiga_inscription_info(outputs_data[tx_index][index]) nft_token_attr[:full_name] = info[:name] nft_token_attr[:symbol] = info[:symbol] nft_token_attr[:decimal] = info[:decimal] nft_token_attr[:published] = true - end - if cell_type == "xudt" + when "xudt" issuer_address = Address.find_by(lock_hash: output.type.args[0..65])&.address_hash items.each_with_index do |output, index| if output.type&.code_hash == CkbSync::Api.instance.unique_cell_code_hash