From c759cc0cf1b1138904296808fdb3f3bfaade1cda Mon Sep 17 00:00:00 2001 From: Miles Zhang Date: Fri, 21 Jun 2024 16:13:49 +0800 Subject: [PATCH] feat: add tags to token collection (#1980) Signed-off-by: Miles Zhang --- .../ckb_sync/new_node_data_processor.rb | 3 +- app/models/token_collection.rb | 5 ++ .../token_collection_serializer.rb | 2 +- app/workers/token_collection_tag_worker.rb | 57 +++++++++++++++++ app/workers/token_transfer_detect_worker.rb | 3 + ...0620083123_add_tags_to_token_collection.rb | 6 ++ db/structure.sql | 62 ++----------------- lib/scheduler.rb | 4 ++ .../api/v2/nft/collections_controller_test.rb | 3 +- .../token_collection_tag_worker_test.rb | 36 +++++++++++ 10 files changed, 121 insertions(+), 60 deletions(-) create mode 100644 app/workers/token_collection_tag_worker.rb create mode 100644 db/migrate/20240620083123_add_tags_to_token_collection.rb create mode 100644 test/workers/token_collection_tag_worker_test.rb diff --git a/app/models/ckb_sync/new_node_data_processor.rb b/app/models/ckb_sync/new_node_data_processor.rb index 5ae92027a..f0368aa5b 100644 --- a/app/models/ckb_sync/new_node_data_processor.rb +++ b/app/models/ckb_sync/new_node_data_processor.rb @@ -630,10 +630,11 @@ def build_udts!(local_block, outputs, outputs_data) if m_nft_class_type.present? m_nft_class_cell = m_nft_class_type.cell_outputs.last parsed_class_data = CkbUtils.parse_token_class_data(m_nft_class_cell.data) - coll = TokenCollection.find_or_create_by( + TokenCollection.find_or_create_by( standard: "m_nft", name: parsed_class_data.name, cell_id: m_nft_class_cell.id, + block_timestamp: m_nft_class_cell.block_timestamp, icon_url: parsed_class_data.renderer, creator_id: m_nft_class_cell.address_id, ) diff --git a/app/models/token_collection.rb b/app/models/token_collection.rb index de46cdc23..6fa7d8b43 100644 --- a/app/models/token_collection.rb +++ b/app/models/token_collection.rb @@ -1,4 +1,6 @@ class TokenCollection < ApplicationRecord + VALID_TAGS = ["invalid", "suspicious", "out-of-length-range", "rgbpp-compatible", "layer-1-asset", "duplicate", "layer-2-asset"] + enum standard: { cota: "cota", spore: "spore", m_nft: "m_nft", nrc721: "nrc721" } has_many :items, class_name: "TokenItem", foreign_key: :collection_id @@ -40,6 +42,7 @@ def as_json(_options = {}) type_script: type_script&.as_json, timestamp: cell&.block_timestamp, sn:, + tags:, } end @@ -184,6 +187,8 @@ def update_h24_ckb_transactions_count # type_script_id :integer # sn :string # h24_ckb_transactions_count :bigint default(0) +# tags :string default([]), is an Array +# block_timestamp :bigint # # Indexes # diff --git a/app/serializers/token_collection_serializer.rb b/app/serializers/token_collection_serializer.rb index 014d80be3..07e9b89d6 100644 --- a/app/serializers/token_collection_serializer.rb +++ b/app/serializers/token_collection_serializer.rb @@ -1,5 +1,5 @@ class TokenCollectionSerializer include FastJsonapi::ObjectSerializer - attributes :standard, :name, :description, :icon_url, :symbol, :sn + attributes :standard, :name, :description, :icon_url, :symbol, :sn, :tags end diff --git a/app/workers/token_collection_tag_worker.rb b/app/workers/token_collection_tag_worker.rb new file mode 100644 index 000000000..d03a4191f --- /dev/null +++ b/app/workers/token_collection_tag_worker.rb @@ -0,0 +1,57 @@ +class TokenCollectionTagWorker + include Sidekiq::Job + + def perform + token_collections = TokenCollection.preload(:creator).where(tags: []).where.not("name IS NULL OR name = ''").limit(100) + unless token_collections.empty? + attrs = + token_collections.map do |token_collection| + tags = mark_tags(token_collection) + { id: token_collection.id, tags: } + end + + TokenCollection.upsert_all(attrs, unique_by: :id, on_duplicate: :update, update_only: :tags) + end + end + + def mark_tags(token_collection) + if invalid_char?(token_collection.name) + ["invalid"] + elsif invisible_char?(token_collection.name) + ["suspicious"] + elsif out_of_length?(token_collection.name) + ["out-of-length-range"] + elsif first_token_collection?(token_collection.name, token_collection.block_timestamp, token_collection.standard) + if rgbpp_lock?(token_collection.creator.address_hash) + ["rgbpp-compatible", "layer-1-asset"] + else + ["rgbpp-compatible", "layer-2-asset"] + end + elsif rgbpp_lock?(token_collection.creator.address_hash) + ["duplicate", "layer-1-asset"] + else + ["duplicate", "layer-2-asset"] + end + end + + def invalid_char?(name) + !name.ascii_only? + end + + def invisible_char?(name) + (name =~ /^[\x21-\x7E]+$/).nil? + end + + def out_of_length?(name) + name.length > 255 + end + + def first_token_collection?(name, block_timestamp, standard) + !TokenCollection.where(name:, standard:).where("block_timestamp < ?", block_timestamp).exists? + end + + def rgbpp_lock?(issuer_address) + address_code_hash = CkbUtils.parse_address(issuer_address).script.code_hash + issuer_address.present? && CkbSync::Api.instance.rgbpp_code_hash.include?(address_code_hash) + end +end diff --git a/app/workers/token_transfer_detect_worker.rb b/app/workers/token_transfer_detect_worker.rb index 79092c8e8..ff9fcd484 100644 --- a/app/workers/token_transfer_detect_worker.rb +++ b/app/workers/token_transfer_detect_worker.rb @@ -111,6 +111,7 @@ def find_or_create_nrc_721_collection(_cell, type_script) if coll.cell_id.blank? || (coll.cell_id < nrc_721_factory_cell.id) coll.update( cell_id: nrc_721_factory_cell.id, + block_timestamp: nrc_721_factory_cell.block_timestamp, symbol: nrc_721_factory_cell.symbol.to_s[0, 16], name: nrc_721_factory_cell.name, icon_url: nrc_721_factory_cell.base_token_uri, @@ -133,6 +134,7 @@ def find_or_create_m_nft_collection(_cell, type_script) if m_nft_class_cell.present? && (coll.cell_id.blank? || (coll.cell_id < m_nft_class_cell.id)) parsed_class_data = CkbUtils.parse_token_class_data(m_nft_class_cell.data) coll.cell_id = m_nft_class_cell.id + coll.block_timestamp = m_nft_class_cell.block_timestamp coll.icon_url = parsed_class_data.renderer coll.name = parsed_class_data.name coll.description = parsed_class_data.description @@ -167,6 +169,7 @@ def find_or_create_spore_collection(_cell, type_script) coll.type_script_id = spore_cluster_cell.type_script_id coll.creator_id = spore_cluster_cell.address_id coll.cell_id = spore_cluster_cell.id + coll.block_timestamp = spore_cluster_cell.block_timestamp coll.name = parsed_cluster_data[:name] coll.description = parsed_cluster_data[:description] coll.save diff --git a/db/migrate/20240620083123_add_tags_to_token_collection.rb b/db/migrate/20240620083123_add_tags_to_token_collection.rb new file mode 100644 index 000000000..1e3550262 --- /dev/null +++ b/db/migrate/20240620083123_add_tags_to_token_collection.rb @@ -0,0 +1,6 @@ +class AddTagsToTokenCollection < ActiveRecord::Migration[7.0] + def change + add_column :token_collections, :tags, :string, array: true, default: [] + add_column :token_collections, :block_timestamp, :bigint + end +end diff --git a/db/structure.sql b/db/structure.sql index d568a38d3..f3527c8df 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -658,39 +658,6 @@ CREATE SEQUENCE public.bitcoin_statistics_id_seq ALTER SEQUENCE public.bitcoin_statistics_id_seq OWNED BY public.bitcoin_statistics.id; --- --- Name: bitcoin_time_locks; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.bitcoin_time_locks ( - id bigint NOT NULL, - bitcoin_transaction_id bigint, - ckb_transaction_id bigint, - cell_output_id bigint, - created_at timestamp(6) without time zone NOT NULL, - updated_at timestamp(6) without time zone NOT NULL -); - - --- --- Name: bitcoin_time_locks_id_seq; Type: SEQUENCE; Schema: public; Owner: - --- - -CREATE SEQUENCE public.bitcoin_time_locks_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - --- --- Name: bitcoin_time_locks_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - --- - -ALTER SEQUENCE public.bitcoin_time_locks_id_seq OWNED BY public.bitcoin_time_locks.id; - - -- -- Name: bitcoin_transactions; Type: TABLE; Schema: public; Owner: - -- @@ -2193,7 +2160,9 @@ CREATE TABLE public.token_collections ( verified boolean DEFAULT false, type_script_id integer, sn character varying, - h24_ckb_transactions_count bigint DEFAULT 0 + h24_ckb_transactions_count bigint DEFAULT 0, + tags character varying[] DEFAULT '{}'::character varying[], + block_timestamp bigint ); @@ -2747,13 +2716,6 @@ ALTER TABLE ONLY public.bitcoin_annotations ALTER COLUMN id SET DEFAULT nextval( ALTER TABLE ONLY public.bitcoin_statistics ALTER COLUMN id SET DEFAULT nextval('public.bitcoin_statistics_id_seq'::regclass); --- --- Name: bitcoin_time_locks id; Type: DEFAULT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.bitcoin_time_locks ALTER COLUMN id SET DEFAULT nextval('public.bitcoin_time_locks_id_seq'::regclass); - - -- -- Name: bitcoin_transactions id; Type: DEFAULT; Schema: public; Owner: - -- @@ -3154,14 +3116,6 @@ ALTER TABLE ONLY public.bitcoin_statistics ADD CONSTRAINT bitcoin_statistics_pkey PRIMARY KEY (id); --- --- Name: bitcoin_time_locks bitcoin_time_locks_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.bitcoin_time_locks - ADD CONSTRAINT bitcoin_time_locks_pkey PRIMARY KEY (id); - - -- -- Name: bitcoin_transactions bitcoin_transactions_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -3944,13 +3898,6 @@ CREATE UNIQUE INDEX index_bitcoin_annotations_on_ckb_transaction_id ON public.bi CREATE UNIQUE INDEX index_bitcoin_statistics_on_timestamp ON public.bitcoin_statistics USING btree ("timestamp"); --- --- Name: index_bitcoin_time_locks_on_cell; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX index_bitcoin_time_locks_on_cell ON public.bitcoin_time_locks USING btree (bitcoin_transaction_id, cell_output_id); - - -- -- Name: index_bitcoin_transactions_on_txid; Type: INDEX; Schema: public; Owner: - -- @@ -5233,6 +5180,7 @@ INSERT INTO "schema_migrations" (version) VALUES ('20240429102325'), ('20240507041552'), ('20240509074313'), -('20240513055849'); +('20240513055849'), +('20240620083123'); diff --git a/lib/scheduler.rb b/lib/scheduler.rb index 149bd6571..a6f355404 100644 --- a/lib/scheduler.rb +++ b/lib/scheduler.rb @@ -129,4 +129,8 @@ def call_worker(clz) call_worker XudtTagWorker end +s.every "5m", overlap: false do + call_worker TokenCollectionTagWorker +end + s.join diff --git a/test/controllers/api/v2/nft/collections_controller_test.rb b/test/controllers/api/v2/nft/collections_controller_test.rb index 782a09675..53e9701c2 100644 --- a/test/controllers/api/v2/nft/collections_controller_test.rb +++ b/test/controllers/api/v2/nft/collections_controller_test.rb @@ -4,12 +4,13 @@ module Api module V2 class NFT::CollectionsControllerTest < ActionDispatch::IntegrationTest test "should get index" do - create :token_collection, name: "token1" + create :token_collection, name: "token1", tags: ["invalid"] create :token_collection, name: "token2" get api_v2_nft_collections_url assert_response :success assert_equal JSON.parse(response.body)["data"].size, 2 + assert_equal ["invalid"], JSON.parse(response.body)["data"].last["tags"] end test "sort by block_timestamp asc" do diff --git a/test/workers/token_collection_tag_worker_test.rb b/test/workers/token_collection_tag_worker_test.rb new file mode 100644 index 000000000..127ae21b5 --- /dev/null +++ b/test/workers/token_collection_tag_worker_test.rb @@ -0,0 +1,36 @@ +require "test_helper" + +class TokenCollectionTagWorkerTest < ActiveJob::TestCase + setup do + @address = create(:address, address_hash: "ckb1qz7xc452rgxs5z0ks3xun46dmdp58sepg0ljtae8ck0d7nah945nvqgqqqqqqx3l3v4") + tx = create(:ckb_transaction) + @cell = create(:cell_output, address_id: @address.id, ckb_transaction_id: tx.id, tx_hash: tx.tx_hash) + end + + test "add invalid tag to token_collection" do + create(:token_collection, name: "ΓΌ", cell_id: @cell.id, creator_id: @address.id) + TokenCollectionTagWorker.new.perform + assert_equal ["invalid"], TokenCollection.last.tags + end + + test "add suspicious tag to token_collection" do + create(:token_collection, name: "CK BB", cell_id: @cell.id, creator_id: @address.id) + TokenCollectionTagWorker.new.perform + assert_equal ["suspicious"], TokenCollection.last.tags + end + + test "add out-of-length-range tag to token_collection" do + create(:token_collection, name: "C" * 256, cell_id: @cell.id, creator_id: @address.id) + TokenCollectionTagWorker.new.perform + assert_equal ["out-of-length-range"], TokenCollection.last.tags + end + + test "add duplicate tag to token_collection" do + create(:token_collection, name: "CKBNFT", cell_id: @cell.id, creator_id: @address.id, block_timestamp: 1.hour.ago.to_i, tags: ["rgbpp-compatible", "layer-1-asset"]) + new_tx = create(:ckb_transaction) + new_cell = create(:cell_output, address_id: @address.id, ckb_transaction_id: new_tx.id, tx_hash: new_tx.tx_hash) + create(:token_collection, name: "CKBNFT", cell_id: new_cell.id, creator_id: @address.id, block_timestamp: Time.now.to_i) + TokenCollectionTagWorker.new.perform + assert_equal ["duplicate", "layer-1-asset"], TokenCollection.last.tags + end +end