{"payload":{"feedbackUrl":"https://github.com/orgs/community/discussions/53140","repo":{"id":621697597,"defaultBranch":"main","name":"tokenizers","ownerLogin":"daulet","currentUserCanPush":false,"isFork":false,"isEmpty":false,"createdAt":"2023-03-31T07:40:07.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/988018?v=4","public":true,"private":false,"isOrgOwned":false},"refInfo":{"name":"","listCacheKey":"v0:1723246625.0","currentOid":""},"activityList":{"items":[{"before":"f5ef49ce7d17cbcdd60c809ef77554be84e95d94","after":"eafe2ce1db50dcd78b3b7157bf2c669109f4e322","ref":"refs/heads/main","pushedAt":"2024-08-09T23:52:50.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"daulet","name":"Daulet Zhanguzin","path":"/daulet","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/988018?s=80&v=4"},"commit":{"message":"fix: update example to latest release","shortMessageHtmlLink":"fix: update example to latest release"}},{"before":"9c7a1e420428e0988e4a5b9e7da7bd6e63f19019","after":null,"ref":"refs/heads/daulet/bazel","pushedAt":"2024-08-09T23:37:05.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"daulet","name":"Daulet Zhanguzin","path":"/daulet","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/988018?s=80&v=4"}},{"before":"8ffa76c427b65ac3d385b4ffd7ba6250247220ea","after":null,"ref":"refs/heads/daulet/v0.20","pushedAt":"2024-08-09T23:37:03.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"daulet","name":"Daulet Zhanguzin","path":"/daulet","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/988018?s=80&v=4"}},{"before":"89e504d5c96f7bf08225f18e00568544e509b936","after":"f5ef49ce7d17cbcdd60c809ef77554be84e95d94","ref":"refs/heads/main","pushedAt":"2024-08-09T23:20:00.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"daulet","name":"Daulet Zhanguzin","path":"/daulet","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/988018?s=80&v=4"},"commit":{"message":"fix: release image","shortMessageHtmlLink":"fix: release image"}},{"before":"7bb47dd52e68ae3349c0461d494921d6a07f7181","after":"89e504d5c96f7bf08225f18e00568544e509b936","ref":"refs/heads/main","pushedAt":"2024-08-09T23:15:14.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"daulet","name":"Daulet Zhanguzin","path":"/daulet","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/988018?s=80&v=4"},"commit":{"message":"Update to huggingface/tokenizers v0.20.0 (#23)\n\n* update tokenizers lib to v0.20.0\r\n\r\n* benchmark new release\r\n\r\n* update bazel builds","shortMessageHtmlLink":"Update to huggingface/tokenizers v0.20.0 (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2458777394\" data-permission-text=\"Title is private\" data-url=\"https://github.com/daulet/tokenizers/issues/23\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/daulet/tokenizers/pull/23/hovercard\" href=\"https://github.com/daulet/tokenizers/pull/23\">#23</a>)"}},{"before":"192f4bd7d8d5e3e77e79971791d01a44ac5acb12","after":"8ffa76c427b65ac3d385b4ffd7ba6250247220ea","ref":"refs/heads/daulet/v0.20","pushedAt":"2024-08-09T23:07:13.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"daulet","name":"Daulet Zhanguzin","path":"/daulet","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/988018?s=80&v=4"},"commit":{"message":"update bazel builds","shortMessageHtmlLink":"update bazel builds"}},{"before":"1b502b65573ea00125eac62fa301c480402be19c","after":"192f4bd7d8d5e3e77e79971791d01a44ac5acb12","ref":"refs/heads/daulet/v0.20","pushedAt":"2024-08-09T23:06:22.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"daulet","name":"Daulet Zhanguzin","path":"/daulet","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/988018?s=80&v=4"},"commit":{"message":"benchmark new release","shortMessageHtmlLink":"benchmark new release"}},{"before":null,"after":"1b502b65573ea00125eac62fa301c480402be19c","ref":"refs/heads/daulet/v0.20","pushedAt":"2024-08-09T22:59:11.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"daulet","name":"Daulet Zhanguzin","path":"/daulet","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/988018?s=80&v=4"},"commit":{"message":"update tokenizers lib to v0.20.0","shortMessageHtmlLink":"update tokenizers lib to v0.20.0"}},{"before":"d9aff87d16f3db537ee005fb45ebca26049e7916","after":"7bb47dd52e68ae3349c0461d494921d6a07f7181","ref":"refs/heads/main","pushedAt":"2024-08-09T21:38:38.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"daulet","name":"Daulet Zhanguzin","path":"/daulet","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/988018?s=80&v=4"},"commit":{"message":"feat: add option to retrieve offsets from tokenizer (#21)\n\n* feat: add option to retrieve offsets from tokenizer\r\n\r\n* fix: avoid tuple struct in offsets\r\n\r\n* fix: free offsets array\r\n\r\n* fix: revert freeing of memory","shortMessageHtmlLink":"feat: add option to retrieve offsets from tokenizer (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2413658784\" data-permission-text=\"Title is private\" data-url=\"https://github.com/daulet/tokenizers/issues/21\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/daulet/tokenizers/pull/21/hovercard\" href=\"https://github.com/daulet/tokenizers/pull/21\">#21</a>)"}},{"before":"fce4681bef8b9b7fc40b0f861fb72e88cba661c4","after":"d9aff87d16f3db537ee005fb45ebca26049e7916","ref":"refs/heads/main","pushedAt":"2024-07-14T05:39:33.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"daulet","name":"Daulet Zhanguzin","path":"/daulet","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/988018?s=80&v=4"},"commit":{"message":"Add tiktoken vs tokenizers bench","shortMessageHtmlLink":"Add tiktoken vs tokenizers bench"}},{"before":"867bf6256404fa578d19fb249785c0234aecbe9c","after":"fce4681bef8b9b7fc40b0f861fb72e88cba661c4","ref":"refs/heads/main","pushedAt":"2024-07-10T15:24:27.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"daulet","name":"Daulet Zhanguzin","path":"/daulet","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/988018?s=80&v=4"},"commit":{"message":"improve rust bench to exclude drop time","shortMessageHtmlLink":"improve rust bench to exclude drop time"}},{"before":"217df12c164da67a24fea94c7d9af6332c616f03","after":"867bf6256404fa578d19fb249785c0234aecbe9c","ref":"refs/heads/main","pushedAt":"2024-07-10T15:14:15.000Z","pushType":"push","commitsCount":3,"pusher":{"login":"daulet","name":"Daulet Zhanguzin","path":"/daulet","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/988018?s=80&v=4"},"commit":{"message":"fix Bazel build","shortMessageHtmlLink":"fix Bazel build"}},{"before":"a735742655834defa37dba15e22a6e58811cc768","after":"217df12c164da67a24fea94c7d9af6332c616f03","ref":"refs/heads/main","pushedAt":"2024-07-09T07:52:28.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"daulet","name":"Daulet Zhanguzin","path":"/daulet","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/988018?s=80&v=4"},"commit":{"message":"upgrade to 0.19.1","shortMessageHtmlLink":"upgrade to 0.19.1"}},{"before":"ba0db9d26089c0edd1352559a0bb1cc2f38cf4f7","after":"a735742655834defa37dba15e22a6e58811cc768","ref":"refs/heads/main","pushedAt":"2024-07-09T07:48:15.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"daulet","name":"Daulet Zhanguzin","path":"/daulet","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/988018?s=80&v=4"},"commit":{"message":"update benchmarks before upgrade to 0.19.1","shortMessageHtmlLink":"update benchmarks before upgrade to 0.19.1"}},{"before":"d503b5b0fc1637e15d5d0cf3645fbf08a3db1458","after":"ba0db9d26089c0edd1352559a0bb1cc2f38cf4f7","ref":"refs/heads/main","pushedAt":"2024-07-09T07:44:27.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"daulet","name":"Daulet Zhanguzin","path":"/daulet","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/988018?s=80&v=4"},"commit":{"message":"update README with linker param requirement","shortMessageHtmlLink":"update README with linker param requirement"}},{"before":"3a615e667565b46ffbc904ff75473a28128da955","after":"d503b5b0fc1637e15d5d0cf3645fbf08a3db1458","ref":"refs/heads/main","pushedAt":"2024-06-12T01:36:13.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"daulet","name":"Daulet Zhanguzin","path":"/daulet","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/988018?s=80&v=4"},"commit":{"message":"Update to allow for platform dependent libs in CGO (#18)\n\n* Update to allow for platform dependent libs in CGO\r\n\r\n* update make test","shortMessageHtmlLink":"Update to allow for platform dependent libs in CGO (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2334301123\" data-permission-text=\"Title is private\" data-url=\"https://github.com/daulet/tokenizers/issues/18\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/daulet/tokenizers/pull/18/hovercard\" href=\"https://github.com/daulet/tokenizers/pull/18\">#18</a>)"}},{"before":"b14c306be9c9b40db4199c7632cd7717ecad7517","after":"3a615e667565b46ffbc904ff75473a28128da955","ref":"refs/heads/main","pushedAt":"2024-04-10T23:25:09.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"daulet","name":"Daulet Zhanguzin","path":"/daulet","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/988018?s=80&v=4"},"commit":{"message":"expose encodeSpecialTokens functionality","shortMessageHtmlLink":"expose encodeSpecialTokens functionality"}},{"before":"8531b2b902c0a5f0092c6f2214cf16657091f3b7","after":"b14c306be9c9b40db4199c7632cd7717ecad7517","ref":"refs/heads/main","pushedAt":"2024-04-10T17:22:47.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"daulet","name":"Daulet Zhanguzin","path":"/daulet","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/988018?s=80&v=4"},"commit":{"message":"update to tokenizers v0.15.2","shortMessageHtmlLink":"update to tokenizers v0.15.2"}},{"before":"6e58579fd74a9ceff9752302683c4021c722af76","after":"8531b2b902c0a5f0092c6f2214cf16657091f3b7","ref":"refs/heads/main","pushedAt":"2024-01-07T00:37:03.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"daulet","name":"Daulet Zhanguzin","path":"/daulet","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/988018?s=80&v=4"},"commit":{"message":"exclude release from bazel build as it is not compatible","shortMessageHtmlLink":"exclude release from bazel build as it is not compatible"}},{"before":"4fd5a669ba280de1c61e6e2b7c87de7d06a0e15c","after":"6e58579fd74a9ceff9752302683c4021c722af76","ref":"refs/heads/main","pushedAt":"2024-01-07T00:24:32.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"daulet","name":"Daulet Zhanguzin","path":"/daulet","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/988018?s=80&v=4"},"commit":{"message":"fix cross compilation: libc::c_char is platform dependent","shortMessageHtmlLink":"fix cross compilation: libc::c_char is platform dependent"}},{"before":"38a9a14c1c56b113461b0c7350c72de949e23cc2","after":"4fd5a669ba280de1c61e6e2b7c87de7d06a0e15c","ref":"refs/heads/main","pushedAt":"2023-11-19T16:55:56.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"daulet","name":"Daulet Zhanguzin","path":"/daulet","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/988018?s=80&v=4"},"commit":{"message":"add contribution README","shortMessageHtmlLink":"add contribution README"}},{"before":"786da4095f5ca3d598db1236c46401b63874f640","after":"38a9a14c1c56b113461b0c7350c72de949e23cc2","ref":"refs/heads/main","pushedAt":"2023-11-15T15:40:37.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"daulet","name":"Daulet Zhanguzin","path":"/daulet","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/988018?s=80&v=4"},"commit":{"message":"support more attributes from the Encoding structure (#5)\n\n* support more attributes from the Encoding structure\r\n\r\nMiniLM requires the attention mask to perform the mean pooling\r\noperation as can be seen at\r\nhttps://huggingface.co/sentence-transformers/all-MiniLM-L6-v2\r\n\r\n* adapt the example and the readme\r\n\r\n* introduce EncodeWithOptions method for selecting the returned attributes\r\n\r\n* add missing options\r\n\r\n* fix the example and the readme\r\n\r\n* fix benchmarks\r\n\r\n---------\r\n\r\nCo-authored-by: Clément Michaud <cmichaud@aviatrix.com>","shortMessageHtmlLink":"support more attributes from the Encoding structure (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"1817320695\" data-permission-text=\"Title is private\" data-url=\"https://github.com/daulet/tokenizers/issues/5\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/daulet/tokenizers/pull/5/hovercard\" href=\"https://github.com/daulet/tokenizers/pull/5\">#5</a>)"}},{"before":"d2b38924ccbd128242cf535039260d7d25040d03","after":"786da4095f5ca3d598db1236c46401b63874f640","ref":"refs/heads/main","pushedAt":"2023-11-15T15:29:29.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"daulet","name":"Daulet Zhanguzin","path":"/daulet","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/988018?s=80&v=4"},"commit":{"message":"enable triggering CI action manually","shortMessageHtmlLink":"enable triggering CI action manually"}},{"before":"e020e7ef4c7dbdba4e330d5f714d1a3495499543","after":"d2b38924ccbd128242cf535039260d7d25040d03","ref":"refs/heads/main","pushedAt":"2023-11-15T15:16:08.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"daulet","name":"Daulet Zhanguzin","path":"/daulet","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/988018?s=80&v=4"},"commit":{"message":"Update ci.yaml","shortMessageHtmlLink":"Update ci.yaml"}},{"before":"c2f743b818e66b201757dc16560f29744609a745","after":"e020e7ef4c7dbdba4e330d5f714d1a3495499543","ref":"refs/heads/main","pushedAt":"2023-11-09T23:25:23.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"daulet","name":"Daulet Zhanguzin","path":"/daulet","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/988018?s=80&v=4"},"commit":{"message":"rename main go target to just tokenizers to simplify downstream use cases","shortMessageHtmlLink":"rename main go target to just tokenizers to simplify downstream use c…"}},{"before":"5e367fe3a49550f70d7b39ee36dfe05c2b605155","after":"c2f743b818e66b201757dc16560f29744609a745","ref":"refs/heads/main","pushedAt":"2023-11-09T03:18:14.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"daulet","name":"Daulet Zhanguzin","path":"/daulet","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/988018?s=80&v=4"},"commit":{"message":"update docker example","shortMessageHtmlLink":"update docker example"}},{"before":"457ba48699ca926523d8ab5051f706a6683c42c6","after":"5e367fe3a49550f70d7b39ee36dfe05c2b605155","ref":"refs/heads/main","pushedAt":"2023-11-09T01:51:20.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"daulet","name":"Daulet Zhanguzin","path":"/daulet","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/988018?s=80&v=4"},"commit":{"message":"Bazel support (#12)\n\n* build rust\r\n\r\n* go builds\r\n\r\n* tests pass\r\n\r\n* release runs too\r\n\r\n* gazelle formatting\r\n\r\n* add CI\r\n\r\n* check in cargo.lock based on this https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html\r\n\r\n* add linker opts\r\n\r\n* does renaming help?\r\n\r\n* builds\r\n\r\n* upgrade to 0.14.1\r\n\r\n* update release files","shortMessageHtmlLink":"Bazel support (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"1982785053\" data-permission-text=\"Title is private\" data-url=\"https://github.com/daulet/tokenizers/issues/12\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/daulet/tokenizers/pull/12/hovercard\" href=\"https://github.com/daulet/tokenizers/pull/12\">#12</a>)"}},{"before":"8a0003441ac9c9681ec85ad5b321d16f061acc6d","after":"9c7a1e420428e0988e4a5b9e7da7bd6e63f19019","ref":"refs/heads/daulet/bazel","pushedAt":"2023-11-09T01:49:04.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"daulet","name":"Daulet Zhanguzin","path":"/daulet","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/988018?s=80&v=4"},"commit":{"message":"update release files","shortMessageHtmlLink":"update release files"}},{"before":"e3bdf1ddf635844fad41936e07d75278909cfddd","after":"8a0003441ac9c9681ec85ad5b321d16f061acc6d","ref":"refs/heads/daulet/bazel","pushedAt":"2023-11-09T00:13:43.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"daulet","name":"Daulet Zhanguzin","path":"/daulet","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/988018?s=80&v=4"},"commit":{"message":"upgrade to 0.14.1","shortMessageHtmlLink":"upgrade to 0.14.1"}},{"before":"060d9d026e6f760619b0eabfc66d1e64b6619e7f","after":"e3bdf1ddf635844fad41936e07d75278909cfddd","ref":"refs/heads/daulet/bazel","pushedAt":"2023-11-08T23:08:39.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"daulet","name":"Daulet Zhanguzin","path":"/daulet","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/988018?s=80&v=4"},"commit":{"message":"builds","shortMessageHtmlLink":"builds"}}],"hasNextPage":true,"hasPreviousPage":false,"activityType":"all","actor":null,"timePeriod":"all","sort":"DESC","perPage":30,"cursor":"djE6ks8AAAAElrXgOAA","startCursor":null,"endCursor":null}},"title":"Activity · daulet/tokenizers"}