{"payload":{"feedbackUrl":"https://github.com/orgs/community/discussions/53140","repo":{"id":665806822,"defaultBranch":"main","name":"llm-jp-tokenizer","ownerLogin":"llm-jp","currentUserCanPush":false,"isFork":false,"isEmpty":false,"createdAt":"2023-07-13T03:45:17.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/134031702?v=4","public":true,"private":false,"isOrgOwned":true},"refInfo":{"name":"","listCacheKey":"v0:1712969320.0","currentOid":""},"activityList":{"items":[{"before":"5d78a47b92515431315d1286cbbf4b8278d089cd","after":"870a27ce6872e105e4b76cdf2e68c8b7ebfc6a37","ref":"refs/heads/main","pushedAt":"2024-04-13T01:16:36.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"tatHi","name":"Tatsuya Hiraoka","path":"/tatHi","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/16539238?s=80&v=4"},"commit":{"message":"Update README.md\n\nrename ver3.0a2 -> ver3.0b1\r\n(the model is renamed without change)","shortMessageHtmlLink":"Update README.md"}},{"before":"c392887bd94c32a55f320301547b103ef18e6aad","after":"5d78a47b92515431315d1286cbbf4b8278d089cd","ref":"refs/heads/main","pushedAt":"2024-04-13T01:12:26.000Z","pushType":"pr_merge","commitsCount":2,"pusher":{"login":"tatHi","name":"Tatsuya Hiraoka","path":"/tatHi","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/16539238?s=80&v=4"},"commit":{"message":"Merge pull request #24 from llm-jp/Release-ver3.0b1\n\nRelease ver3.0b1","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2241155255\" data-permission-text=\"Title is private\" data-url=\"https://github.com/llm-jp/llm-jp-tokenizer/issues/24\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/llm-jp/llm-jp-tokenizer/pull/24/hovercard\" href=\"https://github.com/llm-jp/llm-jp-tokenizer/pull/24\">#24</a> from llm-jp/Release-ver3.0b1"}},{"before":null,"after":"186940fb414a0251f0723d73ac0f6d72533a457a","ref":"refs/heads/Release-ver3.0b1","pushedAt":"2024-04-13T00:48:40.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"hiroshi-matsuda-rit","name":"Hiroshi Matsuda","path":"/hiroshi-matsuda-rit","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/40782025?s=80&v=4"},"commit":{"message":"Release ver3.0b1","shortMessageHtmlLink":"Release ver3.0b1"}},{"before":"96391ca59562a0ddabdf4ff4cd4c069604d9ddda","after":"c392887bd94c32a55f320301547b103ef18e6aad","ref":"refs/heads/main","pushedAt":"2024-04-07T16:22:52.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"hiroshi-matsuda-rit","name":"Hiroshi Matsuda","path":"/hiroshi-matsuda-rit","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/40782025?s=80&v=4"},"commit":{"message":"revise mask token in v3.0a3","shortMessageHtmlLink":"revise mask token in v3.0a3"}},{"before":"30b49d313e3fc4559ec4fd5c035b1ee8b5d03033","after":"96391ca59562a0ddabdf4ff4cd4c069604d9ddda","ref":"refs/heads/main","pushedAt":"2024-04-06T23:15:50.000Z","pushType":"pr_merge","commitsCount":2,"pusher":{"login":"tatHi","name":"Tatsuya Hiraoka","path":"/tatHi","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/16539238?s=80&v=4"},"commit":{"message":"Merge pull request #23 from vzvu3k6k/patch-1\n\nFix small typos in README.md","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2229372912\" data-permission-text=\"Title is private\" data-url=\"https://github.com/llm-jp/llm-jp-tokenizer/issues/23\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/llm-jp/llm-jp-tokenizer/pull/23/hovercard\" href=\"https://github.com/llm-jp/llm-jp-tokenizer/pull/23\">#23</a> from vzvu3k6k/patch-1"}},{"before":"54c98b0fbcfdf30e914ebfc32617af8923346e42","after":"30b49d313e3fc4559ec4fd5c035b1ee8b5d03033","ref":"refs/heads/main","pushedAt":"2024-04-06T19:36:17.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"hiroshi-matsuda-rit","name":"Hiroshi Matsuda","path":"/hiroshi-matsuda-rit","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/40782025?s=80&v=4"},"commit":{"message":"add llama2 based tokenizer configuration as v3.0a4","shortMessageHtmlLink":"add llama2 based tokenizer configuration as v3.0a4"}},{"before":"373cd4d62d8052c8a9a492d431f1a98da2c1f70a","after":"54c98b0fbcfdf30e914ebfc32617af8923346e42","ref":"refs/heads/main","pushedAt":"2024-03-29T17:44:08.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"hiroshi-matsuda-rit","name":"Hiroshi Matsuda","path":"/hiroshi-matsuda-rit","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/40782025?s=80&v=4"},"commit":{"message":"add --eod-id argument","shortMessageHtmlLink":"add --eod-id argument"}},{"before":"bdb8efc6b160375fc261d48c7b755f64182406d1","after":"373cd4d62d8052c8a9a492d431f1a98da2c1f70a","ref":"refs/heads/main","pushedAt":"2024-03-29T17:19:53.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"hiroshi-matsuda-rit","name":"Hiroshi Matsuda","path":"/hiroshi-matsuda-rit","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/40782025?s=80&v=4"},"commit":{"message":"revise eos token in post_processor","shortMessageHtmlLink":"revise eos token in post_processor"}},{"before":"69bfdbe7d750bb90407296ce352d30d4847a38e7","after":"bdb8efc6b160375fc261d48c7b755f64182406d1","ref":"refs/heads/main","pushedAt":"2024-03-25T13:28:43.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"tatHi","name":"Tatsuya Hiraoka","path":"/tatHi","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/16539238?s=80&v=4"},"commit":{"message":"Update README.md","shortMessageHtmlLink":"Update README.md"}},{"before":"789a5df8f9e209a24fc1f2e3a4cabfd8c7b8f180","after":"69bfdbe7d750bb90407296ce352d30d4847a38e7","ref":"refs/heads/main","pushedAt":"2024-03-09T13:54:31.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"hiroshi-matsuda-rit","name":"Hiroshi Matsuda","path":"/hiroshi-matsuda-rit","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/40782025?s=80&v=4"},"commit":{"message":"add hf ver3.0a2","shortMessageHtmlLink":"add hf ver3.0a2"}},{"before":"f162bc1f65651d443543af97e1ea398ba12da213","after":"789a5df8f9e209a24fc1f2e3a4cabfd8c7b8f180","ref":"refs/heads/main","pushedAt":"2024-03-01T09:40:46.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"tatHi","name":"Tatsuya Hiraoka","path":"/tatHi","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/16539238?s=80&v=4"},"commit":{"message":"Update README.md\n\nSentencePiece v0.2.0で未検証である旨を記載しました","shortMessageHtmlLink":"Update README.md"}},{"before":"991c07953cd37d3d6a9eaaa46227cd023591bcc3","after":"f162bc1f65651d443543af97e1ea398ba12da213","ref":"refs/heads/main","pushedAt":"2024-03-01T07:35:54.000Z","pushType":"pr_merge","commitsCount":8,"pusher":{"login":"tatHi","name":"Tatsuya Hiraoka","path":"/tatHi","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/16539238?s=80&v=4"},"commit":{"message":"Merge pull request #22 from llm-jp/tokenizer-v3.0a\n\nTokenizer v3.0a","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2162802662\" data-permission-text=\"Title is private\" data-url=\"https://github.com/llm-jp/llm-jp-tokenizer/issues/22\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/llm-jp/llm-jp-tokenizer/pull/22/hovercard\" href=\"https://github.com/llm-jp/llm-jp-tokenizer/pull/22\">#22</a> from llm-jp/tokenizer-v3.0a"}},{"before":"129dadcbfe42386f66c2e69f82de27db29b9ce40","after":"7d69c27e0f87af43c19780138eebfe17b9b96e9f","ref":"refs/heads/tokenizer-v3.0a","pushedAt":"2024-03-01T07:33:32.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"tatHi","name":"Tatsuya Hiraoka","path":"/tatHi","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/16539238?s=80&v=4"},"commit":{"message":"remove file","shortMessageHtmlLink":"remove file"}},{"before":"2a0423c0716322caee4c3b3cd338e01553cbd509","after":"129dadcbfe42386f66c2e69f82de27db29b9ce40","ref":"refs/heads/tokenizer-v3.0a","pushedAt":"2024-03-01T07:28:11.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"tatHi","name":"Tatsuya Hiraoka","path":"/tatHi","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/16539238?s=80&v=4"},"commit":{"message":"remove models","shortMessageHtmlLink":"remove models"}},{"before":"c5f87a208f1dd81fc13f2b240ac0b27fe5727629","after":"2a0423c0716322caee4c3b3cd338e01553cbd509","ref":"refs/heads/tokenizer-v3.0a","pushedAt":"2024-03-01T07:18:22.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"tatHi","name":"Tatsuya Hiraoka","path":"/tatHi","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/16539238?s=80&v=4"},"commit":{"message":"revise readme","shortMessageHtmlLink":"revise readme"}},{"before":"441b0bea941c278faf584ae47576609652e55193","after":"c5f87a208f1dd81fc13f2b240ac0b27fe5727629","ref":"refs/heads/tokenizer-v3.0a","pushedAt":"2024-03-01T07:16:01.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"tatHi","name":"Tatsuya Hiraoka","path":"/tatHi","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/16539238?s=80&v=4"},"commit":{"message":"revise README for v3","shortMessageHtmlLink":"revise README for v3"}},{"before":"8b556ccb3bdfcbbef8e346865c67e98c1e435d6b","after":"991c07953cd37d3d6a9eaaa46227cd023591bcc3","ref":"refs/heads/main","pushedAt":"2024-03-01T01:11:53.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"hiroshi-matsuda-rit","name":"Hiroshi Matsuda","path":"/hiroshi-matsuda-rit","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/40782025?s=80&v=4"},"commit":{"message":"add ver3.0a2 archives in models","shortMessageHtmlLink":"add ver3.0a2 archives in models"}},{"before":"5071ebe4d63ba29a66ddfad508091dfd2934499c","after":"99793e2d2ce2e3eea3a95effd2738fba709cd08f","ref":"refs/heads/tokenizer-v3.0a_train_vocab","pushedAt":"2024-02-26T14:08:53.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"hiroshi-matsuda-rit","name":"Hiroshi Matsuda","path":"/hiroshi-matsuda-rit","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/40782025?s=80&v=4"},"commit":{"message":"add filter_vocab.py","shortMessageHtmlLink":"add filter_vocab.py"}},{"before":"817fb1904e9ee2b9f9b8a0e78c125fc2b6ca57e6","after":"441b0bea941c278faf584ae47576609652e55193","ref":"refs/heads/tokenizer-v3.0a","pushedAt":"2024-02-26T05:01:44.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"tatHi","name":"Tatsuya Hiraoka","path":"/tatHi","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/16539238?s=80&v=4"},"commit":{"message":"add v3a models","shortMessageHtmlLink":"add v3a models"}},{"before":"f45c014bbef278c22f78da7296aafca28429834a","after":"5071ebe4d63ba29a66ddfad508091dfd2934499c","ref":"refs/heads/tokenizer-v3.0a_train_vocab","pushedAt":"2024-02-16T07:46:55.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"hiroshi-matsuda-rit","name":"Hiroshi Matsuda","path":"/hiroshi-matsuda-rit","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/40782025?s=80&v=4"},"commit":{"message":"add README.md","shortMessageHtmlLink":"add README.md"}},{"before":null,"after":"f45c014bbef278c22f78da7296aafca28429834a","ref":"refs/heads/tokenizer-v3.0a_train_vocab","pushedAt":"2024-02-16T07:36:30.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"hiroshi-matsuda-rit","name":"Hiroshi Matsuda","path":"/hiroshi-matsuda-rit","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/40782025?s=80&v=4"},"commit":{"message":"add scripts_v3/train_vocab/","shortMessageHtmlLink":"add scripts_v3/train_vocab/"}},{"before":null,"after":"817fb1904e9ee2b9f9b8a0e78c125fc2b6ca57e6","ref":"refs/heads/tokenizer-v3.0a","pushedAt":"2024-01-29T12:30:04.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"tatHi","name":"Tatsuya Hiraoka","path":"/tatHi","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/16539238?s=80&v=4"},"commit":{"message":"init v3 alpha","shortMessageHtmlLink":"init v3 alpha"}},{"before":"56ab5f3472597c29aeccdad50778d96c3aa7409d","after":"8b556ccb3bdfcbbef8e346865c67e98c1e435d6b","ref":"refs/heads/main","pushedAt":"2023-12-20T11:11:42.000Z","pushType":"pr_merge","commitsCount":2,"pusher":{"login":"hiroshi-matsuda-rit","name":"Hiroshi Matsuda","path":"/hiroshi-matsuda-rit","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/40782025?s=80&v=4"},"commit":{"message":"Merge pull request #21 from llm-jp/fix_empty_string_entry\n\nbugfix for empty string entries","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2050339692\" data-permission-text=\"Title is private\" data-url=\"https://github.com/llm-jp/llm-jp-tokenizer/issues/21\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/llm-jp/llm-jp-tokenizer/pull/21/hovercard\" href=\"https://github.com/llm-jp/llm-jp-tokenizer/pull/21\">#21</a> from llm-jp/fix_empty_string_entry"}},{"before":null,"after":"3488e7e7aff918805c2378ff5d4c76605eec7147","ref":"refs/heads/fix_empty_string_entry","pushedAt":"2023-12-20T11:10:11.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"hiroshi-matsuda-rit","name":"Hiroshi Matsuda","path":"/hiroshi-matsuda-rit","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/40782025?s=80&v=4"},"commit":{"message":"bugfix for empty string entries","shortMessageHtmlLink":"bugfix for empty string entries"}},{"before":"49f555459832cb8ed7eef132fd34e6eaa83bd077","after":"56ab5f3472597c29aeccdad50778d96c3aa7409d","ref":"refs/heads/main","pushedAt":"2023-11-19T03:29:53.000Z","pushType":"push","commitsCount":2,"pusher":{"login":"hiroshi-matsuda-rit","name":"Hiroshi Matsuda","path":"/hiroshi-matsuda-rit","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/40782025?s=80&v=4"},"commit":{"message":"hf ver2.2-b3","shortMessageHtmlLink":"hf ver2.2-b3"}},{"before":"e5a3f2f31334a1d5fef1b41f8fe1dab5532ce7ea","after":"49f555459832cb8ed7eef132fd34e6eaa83bd077","ref":"refs/heads/main","pushedAt":"2023-10-20T08:07:35.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"tatHi","name":"Tatsuya Hiraoka","path":"/tatHi","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/16539238?s=80&v=4"},"commit":{"message":"Update howToCreateModel_ver2.md","shortMessageHtmlLink":"Update howToCreateModel_ver2.md"}},{"before":"943407166eeb646ef5620bfbdd88bffed1511d03","after":"e5a3f2f31334a1d5fef1b41f8fe1dab5532ce7ea","ref":"refs/heads/main","pushedAt":"2023-10-20T08:02:30.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"tatHi","name":"Tatsuya Hiraoka","path":"/tatHi","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/16539238?s=80&v=4"},"commit":{"message":"Update README.md (about sp)","shortMessageHtmlLink":"Update README.md (about sp)"}},{"before":"93e7d18f0e4b5c502c7cc950c2e5dac0bc05c01b","after":"943407166eeb646ef5620bfbdd88bffed1511d03","ref":"refs/heads/main","pushedAt":"2023-10-19T18:13:55.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"hiroshi-matsuda-rit","name":"Hiroshi Matsuda","path":"/hiroshi-matsuda-rit","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/40782025?s=80&v=4"},"commit":{"message":"Update README.md","shortMessageHtmlLink":"Update README.md"}},{"before":"52892c7c59cd5afef75677f398f994c67e9a0529","after":"93e7d18f0e4b5c502c7cc950c2e5dac0bc05c01b","ref":"refs/heads/main","pushedAt":"2023-10-19T17:42:46.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"hiroshi-matsuda-rit","name":"Hiroshi Matsuda","path":"/hiroshi-matsuda-rit","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/40782025?s=80&v=4"},"commit":{"message":"Update README.md","shortMessageHtmlLink":"Update README.md"}},{"before":"132f21625417ed0f3dc6484bf0bc1fb6a433acdd","after":"52892c7c59cd5afef75677f398f994c67e9a0529","ref":"refs/heads/main","pushedAt":"2023-10-18T06:58:33.000Z","pushType":"pr_merge","commitsCount":2,"pusher":{"login":"tatHi","name":"Tatsuya Hiraoka","path":"/tatHi","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/16539238?s=80&v=4"},"commit":{"message":"Merge pull request #20 from llm-jp/add-special-tokens\n\nspecial tokenリストの追加です．セルフマージします．","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"1948996889\" data-permission-text=\"Title is private\" data-url=\"https://github.com/llm-jp/llm-jp-tokenizer/issues/20\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/llm-jp/llm-jp-tokenizer/pull/20/hovercard\" href=\"https://github.com/llm-jp/llm-jp-tokenizer/pull/20\">#20</a> from llm-jp/add-special-tokens"}}],"hasNextPage":true,"hasPreviousPage":false,"activityType":"all","actor":null,"timePeriod":"all","sort":"DESC","perPage":30,"cursor":"djE6ks8AAAAEL5I7EgA","startCursor":null,"endCursor":null}},"title":"Activity · llm-jp/llm-jp-tokenizer"}