From 839d256744e74000c0432dab172c2e2bc299f9d1 Mon Sep 17 00:00:00 2001 From: NanZhang Date: Mon, 4 Sep 2023 14:38:53 +0800 Subject: [PATCH 1/2] feat: sync referring cells (#1416) * feat: add referring cells * chore: output renamed to cell * feat: delete referring_cells when revert block --- app/controllers/api/v2/scripts_controller.rb | 106 ++++++++++-------- app/jobs/revert_block_job.rb | 6 + app/models/cell_output.rb | 1 + .../ckb_sync/new_node_data_processor.rb | 2 +- app/models/referring_cell.rb | 39 +++++-- .../v2/scripts/referring_cells.json.jbuilder | 30 +++++ config/routes/v2.rb | 1 + ...29061910_add_indexes_to_referring_cells.rb | 6 + db/structure.sql | 17 ++- .../migration/generate_referring_cells.rake | 26 +++++ .../api/v2/scripts_controller_test.rb | 7 ++ 11 files changed, 181 insertions(+), 60 deletions(-) create mode 100644 app/views/api/v2/scripts/referring_cells.json.jbuilder create mode 100644 db/migrate/20230829061910_add_indexes_to_referring_cells.rb create mode 100644 lib/tasks/migration/generate_referring_cells.rake diff --git a/app/controllers/api/v2/scripts_controller.rb b/app/controllers/api/v2/scripts_controller.rb index d0a8ebf1b..df57f2c86 100644 --- a/app/controllers/api/v2/scripts_controller.rb +++ b/app/controllers/api/v2/scripts_controller.rb @@ -1,65 +1,73 @@ require "jbuilder" -module Api::V2 - class ScriptsController < BaseController - before_action :set_page_and_page_size - before_action :find_script +module Api + module V2 + class ScriptsController < BaseController + before_action :set_page_and_page_size + before_action :find_script - def general_info - head :not_found and return if @script.blank? + def general_info + head :not_found and return if @script.blank? - render json: { - data: get_script_content(@script) - } - end + render json: { + data: get_script_content(@script) + } + end - def ckb_transactions - head :not_found and return if @script.blank? + def ckb_transactions + head :not_found and return if @script.blank? - scope = CellDependency.where(contract_id: @contract.id).order(ckb_transaction_id: :desc) - tx_ids = scope.page(params[:page]).pluck(:ckb_transaction_id) - @ckb_transactions = CkbTransaction.find(tx_ids) - @total = scope.count - end + scope = CellDependency.where(contract_id: @contract.id).order(ckb_transaction_id: :desc) + tx_ids = scope.page(params[:page]).pluck(:ckb_transaction_id) + @ckb_transactions = CkbTransaction.find(tx_ids) + @total = scope.count + end - def deployed_cells - head :not_found and return if @script.blank? || @script.contract.blank? + def deployed_cells + head :not_found and return if @script.blank? || @script.contract.blank? - @deployed_cells = @contract.deployed_cells.page(@page).per(@page_size).fast_page - end + @deployed_cells = @contract.deployed_cells.page(@page).per(@page_size).fast_page + end - private + def referring_cells + head :not_found and return if @script.blank? - def get_script_content(script) - column_name = script.instance_of?(TypeScript) ? "type_script_id" : "lock_script_id" - @my_referring_cells = CellOutput.live.where(column_name => script.id) - @deployed_cells = @contract&.deployed_cell_outputs&.live - { - id: script.id, - code_hash: script.code_hash, - hash_type: script.hash_type, - script_type: script.class.to_s, - capacity_of_deployed_cells: @deployed_cells&.sum(:capacity), - capacity_of_referring_cells: @my_referring_cells.sum(:capacity), - count_of_transactions: @contract&.ckb_transactions&.count.to_i, - count_of_deployed_cells: @deployed_cells&.count.to_i, - count_of_referring_cells: @my_referring_cells.size.to_i - } - end + @referring_cells = @contract.referring_cells.page(@page).per(@page_size).fast_page + end - def set_page_and_page_size - @page = params[:page] || 1 - @page_size = params[:page_size] || 10 - end + private + + def get_script_content(script) + column_name = script.instance_of?(TypeScript) ? "type_script_id" : "lock_script_id" + @my_referring_cells = CellOutput.live.where(column_name => script.id) + @deployed_cells = @contract&.deployed_cell_outputs&.live + { + id: script.id, + code_hash: script.code_hash, + hash_type: script.hash_type, + script_type: script.class.to_s, + capacity_of_deployed_cells: @deployed_cells&.sum(:capacity), + capacity_of_referring_cells: @my_referring_cells.sum(:capacity), + count_of_transactions: @contract&.ckb_transactions&.count.to_i, + count_of_deployed_cells: @deployed_cells&.count.to_i, + count_of_referring_cells: @my_referring_cells.size.to_i + } + end - def find_script - @script = TypeScript.find_by(code_hash: params[:code_hash], - hash_type: params[:hash_type]) - if @script.blank? - @script = LockScript.find_by(code_hash: params[:code_hash], + def set_page_and_page_size + @page = params[:page] || 1 + @page_size = params[:page_size] || 10 + end + + def find_script + @script = TypeScript.find_by(code_hash: params[:code_hash], + hash_type: params[:hash_type]) + if @script.blank? + @script = LockScript.find_by(code_hash: params[:code_hash], + hash_type: params[:hash_type]) + end + @contract = Contract.find_by(code_hash: params[:code_hash], hash_type: params[:hash_type]) end - @contract = Contract.find_by(code_hash: params[:code_hash], - hash_type: params[:hash_type]) end end end diff --git a/app/jobs/revert_block_job.rb b/app/jobs/revert_block_job.rb index 1cb786ec4..49ac115c7 100644 --- a/app/jobs/revert_block_job.rb +++ b/app/jobs/revert_block_job.rb @@ -33,6 +33,7 @@ def perform(local_tip_block = nil) benchmark :recalculate_udt_accounts, udt_type_hashes, local_tip_block benchmark :update_address_balance_and_ckb_transactions_count, local_tip_block benchmark :revert_block_rewards, local_tip_block + benchmark :revert_referring_cells, local_tip_block ForkedEvent.create!(block_number: local_tip_block.number, epoch_number: local_tip_block.epoch, block_timestamp: local_tip_block.timestamp) ApplicationRecord.benchmark "BlockStatisticGenerator" do @@ -208,4 +209,9 @@ def decrease_records_count(local_tip_block) normal_transactions = local_tip_block.ckb_transactions.normal ckb_transaction_counter.decrement!(:count, normal_transactions.count) if normal_transactions.present? end + + def revert_referring_cells(local_tip_block) + tx_ids = local_tip_block.ckb_transaction_ids + ReferringCell.where(ckb_transaction_id: tx_ids).delete_all + end end diff --git a/app/models/cell_output.rb b/app/models/cell_output.rb index 52fcf89a0..b0903d392 100644 --- a/app/models/cell_output.rb +++ b/app/models/cell_output.rb @@ -27,6 +27,7 @@ class CellOutput < ApplicationRecord # the cell_inputs won't always be the same as `consumed_by`.`cell_inputs` has_many :cell_inputs, foreign_key: :previous_cell_output_id belongs_to :deployed_cell, optional: true + has_one :referring_cell # the block_id is actually the same as ckb_transaction.block_id, must be on chain # but one cell may be included by pending transactions, so block_id may be null belongs_to :block, optional: true diff --git a/app/models/ckb_sync/new_node_data_processor.rb b/app/models/ckb_sync/new_node_data_processor.rb index 48b75213e..888c9de09 100644 --- a/app/models/ckb_sync/new_node_data_processor.rb +++ b/app/models/ckb_sync/new_node_data_processor.rb @@ -128,7 +128,7 @@ def check_invalid_address(address) def generate_deployed_cells_and_referring_cells(local_block) local_block.ckb_transactions.each do |ckb_transaction| DeployedCell.create_initial_data_for_ckb_transaction ckb_transaction, tx_cell_deps[ckb_transaction.tx_hash] - # ReferringCell.create_initial_data_for_ckb_transaction ckb_transaction + ReferringCell.create_initial_data_for_ckb_transaction ckb_transaction end end diff --git a/app/models/referring_cell.rb b/app/models/referring_cell.rb index 6083902eb..49f8cf65d 100644 --- a/app/models/referring_cell.rb +++ b/app/models/referring_cell.rb @@ -7,21 +7,37 @@ class ReferringCell < ApplicationRecord # create initial data # please run this script - def self.create_initial_data ckb_transaction_id = nil + def self.create_initial_data(ckb_transaction_id = nil) CkbTransaction.where("id <= ?", ckb_transaction_id).find_each do |ckb_transaction| self.create_initial_data_for_ckb_transaction ckb_transaction end end - def self.create_initial_data_for_ckb_transaction ckb_transaction - ckb_transaction.cell_outputs.each do |cell_output| - contract_id = nil - if cell_output.lock_script_id.present? - contract_id = cell_output.lock_script.contract.id rescue nil - elsif cell_output.type_script_id.present? - contract_id = cell_output.type_script.contract.id rescue nil + def self.create_initial_data_for_ckb_transaction(ckb_transaction) + inputs = ckb_transaction.inputs + outputs = ckb_transaction.outputs + + (inputs + outputs).each do |cell| + contract = cell.lock_script.contract + contract ||= cell.type_script&.contract + + next unless contract + + if cell.live? + ReferringCell.create_or_find_by( + cell_output_id: cell.id, + ckb_transaction_id: ckb_transaction.id, + contract_id: contract.id + ) + elsif cell.dead? + referring_cell = ReferringCell.find_by( + cell_output_id: cell.id, + ckb_transaction_id: ckb_transaction.id, + contract_id: contract.id + ) + + referring_cell.destroy if referring_cell end - ReferringCell.create_or_find_by(cell_output_id: cell_output.id, ckb_transaction_id: ckb_transaction.id, contract_id: contract_id) if contract_id.present? end end end @@ -37,3 +53,8 @@ def self.create_initial_data_for_ckb_transaction ckb_transaction # created_at :datetime not null # updated_at :datetime not null # +# Indexes +# +# index_referring_cells_on_cell_output_id (cell_output_id) UNIQUE +# index_referring_cells_on_contract_id_and_cell_output_id (contract_id,cell_output_id) UNIQUE +# diff --git a/app/views/api/v2/scripts/referring_cells.json.jbuilder b/app/views/api/v2/scripts/referring_cells.json.jbuilder new file mode 100644 index 000000000..3e266f9bf --- /dev/null +++ b/app/views/api/v2/scripts/referring_cells.json.jbuilder @@ -0,0 +1,30 @@ +json.data do + json.referring_cells @referring_cells do |referring_cell| + cell_output = referring_cell.cell_output + json.id cell_output.id + json.capacity cell_output.capacity + json.ckb_transaction_id cell_output.ckb_transaction_id + json.created_at cell_output.created_at + json.updated_at cell_output.updated_at + json.status cell_output.status + json.address_id cell_output.address_id + json.block_id cell_output.block_id + json.tx_hash cell_output.tx_hash + json.cell_index cell_output.cell_index + json.consumed_by_id cell_output.consumed_by_id + json.cell_type cell_output.cell_type + json.data_size cell_output.data_size + json.occupied_capacity cell_output.occupied_capacity + json.block_timestamp cell_output.block_timestamp + json.consumed_block_timestamp cell_output.consumed_block_timestamp + json.type_hash cell_output.type_hash + json.udt_amount cell_output.udt_amount + json.dao cell_output.dao + json.lock_script_id cell_output.lock_script_id + json.type_script_id cell_output.type_script_id + end + json.meta do + json.total @referring_cells.count + json.page_size @page_size.to_i + end +end diff --git a/config/routes/v2.rb b/config/routes/v2.rb index 226d14b54..0d1dfe8cb 100644 --- a/config/routes/v2.rb +++ b/config/routes/v2.rb @@ -58,6 +58,7 @@ collection do get :ckb_transactions get :deployed_cells + get :referring_cells get :general_info end end diff --git a/db/migrate/20230829061910_add_indexes_to_referring_cells.rb b/db/migrate/20230829061910_add_indexes_to_referring_cells.rb new file mode 100644 index 000000000..a062dfd8b --- /dev/null +++ b/db/migrate/20230829061910_add_indexes_to_referring_cells.rb @@ -0,0 +1,6 @@ +class AddIndexesToReferringCells < ActiveRecord::Migration[7.0] + def change + add_index :referring_cells, [:contract_id, :cell_output_id], unique: true + add_index :referring_cells, :cell_output_id, unique: true + end +end diff --git a/db/structure.sql b/db/structure.sql index b5ef00be1..2d66ef4b6 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -3910,6 +3910,20 @@ CREATE INDEX index_pool_transaction_entries_on_tx_hash ON public.pool_transactio CREATE INDEX index_pool_transaction_entries_on_tx_status ON public.pool_transaction_entries USING btree (tx_status); +-- +-- Name: index_referring_cells_on_cell_output_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX index_referring_cells_on_cell_output_id ON public.referring_cells USING btree (cell_output_id); + + +-- +-- Name: index_referring_cells_on_contract_id_and_cell_output_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX index_referring_cells_on_contract_id_and_cell_output_id ON public.referring_cells USING btree (contract_id, cell_output_id); + + -- -- Name: index_reject_reasons_on_ckb_transaction_id; Type: INDEX; Schema: public; Owner: - -- @@ -4669,6 +4683,7 @@ INSERT INTO "schema_migrations" (version) VALUES ('20230630112234'), ('20230711040233'), ('20230802015907'), -('20230808020637'); +('20230808020637'), +('20230829061910'); diff --git a/lib/tasks/migration/generate_referring_cells.rake b/lib/tasks/migration/generate_referring_cells.rake new file mode 100644 index 000000000..7033e1e1c --- /dev/null +++ b/lib/tasks/migration/generate_referring_cells.rake @@ -0,0 +1,26 @@ +namespace :migration do + desc "Usage: RAILS_ENV=production bundle exec rake migration:generate_referring_cells" + task generate_referring_cells: :environment do + live_cells = CellOutput.live.left_joins(:referring_cell).where(referring_cells: { id: nil }) + progress_bar = ProgressBar.create({ total: live_cells.count, format: "%e %B %p%% %c/%C" }) + + live_cells.find_in_batches do |outputs| + outputs.each do |output| + progress_bar.increment + + contract = output.lock_script.contract + contract ||= output.type_script&.contract + + next unless contract + + ReferringCell.create_or_find_by( + cell_output_id: output.id, + ckb_transaction_id: output.ckb_transaction_id, + contract_id: contract.id + ) + end + end + + puts "done" + end +end diff --git a/test/controllers/api/v2/scripts_controller_test.rb b/test/controllers/api/v2/scripts_controller_test.rb index 954c50e4c..81d6aa090 100644 --- a/test/controllers/api/v2/scripts_controller_test.rb +++ b/test/controllers/api/v2/scripts_controller_test.rb @@ -16,6 +16,9 @@ class ScriptsControllerTest < ActionDispatch::IntegrationTest create :deployed_cell, contract_id: @contract.id, cell_output_id: @cell_output1.id create :deployed_cell, contract_id: @contract.id, cell_output_id: @cell_output2.id create :deployed_cell, contract_id: @contract.id, cell_output_id: @cell_output3.id + create :referring_cell, contract_id: @contract.id, cell_output_id: @cell_output1.id, ckb_transaction_id: @cell_output1.ckb_transaction_id + create :referring_cell, contract_id: @contract.id, cell_output_id: @cell_output2.id, ckb_transaction_id: @cell_output2.ckb_transaction_id + create :referring_cell, contract_id: @contract.id, cell_output_id: @cell_output3.id, ckb_transaction_id: @cell_output3.ckb_transaction_id end test "should get ckb_transactions" do @@ -28,6 +31,10 @@ class ScriptsControllerTest < ActionDispatch::IntegrationTest assert_response :success end + test "should get referring_cells" do + valid_get referring_cells_api_v2_scripts_url(code_hash: @code_hash, hash_type: @hash_type) + assert_response :success + end end end end From 4b5edd4e0ecdd70b54ed4a36188f99b31abf5a25 Mon Sep 17 00:00:00 2001 From: NanZhang Date: Mon, 4 Sep 2023 21:00:07 +0800 Subject: [PATCH 2/2] chore: update referring cells total count (#1432) --- app/views/api/v2/scripts/referring_cells.json.jbuilder | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/api/v2/scripts/referring_cells.json.jbuilder b/app/views/api/v2/scripts/referring_cells.json.jbuilder index 3e266f9bf..a4cc5f13f 100644 --- a/app/views/api/v2/scripts/referring_cells.json.jbuilder +++ b/app/views/api/v2/scripts/referring_cells.json.jbuilder @@ -24,7 +24,7 @@ json.data do json.type_script_id cell_output.type_script_id end json.meta do - json.total @referring_cells.count + json.total @referring_cells.total_count json.page_size @page_size.to_i end end