Update spaCy pipeline

README.md

@@ -26,8 +26,8 @@ model-index:
 | **Name** | `de_trf_nrp` |
 | **Version** | `0.0.0` |
 | **spaCy** | `>=3.8.3,<3.9.0` |
-| **Default Pipeline** | `ner_transformer`, `ner`, `merge_entities`, `base_transformer`, `morphologizer`, `tagger`, `parser`, `trainable_lemmatizer` |
-| **Components** | `ner_transformer`, `ner`, `merge_entities`, `base_transformer`, `morphologizer`, `tagger`, `parser`, `trainable_lemmatizer` |
+| **Default Pipeline** | `ner_transformer`, `ner`, `base_transformer`, `morphologizer`, `tagger`, `parser`, `trainable_lemmatizer` |
+| **Components** | `ner_transformer`, `ner`, `base_transformer`, `morphologizer`, `tagger`, `parser`, `trainable_lemmatizer` |
 | **Vectors** | 0 keys, 0 unique vectors (0 dimensions) |
 | **Sources** | n/a |
 | **License** | n/a |

base_transformer/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7c7c7401f3ae369d31fccbbc168d7139b9dab09437344c82baa2778b0482d871
+oid sha256:e2b982402ec283f46b2fdfcac055537916109a1782b6b35369bfded04fcbdee4
 size 440759145

config.cfg

@@ -10,7 +10,7 @@ seed = 17
 
 [nlp]
 lang = "de"
-pipeline = ["ner_transformer","ner","merge_entities","base_transformer","morphologizer","tagger","parser","trainable_lemmatizer"]
+pipeline = ["ner_transformer","ner","base_transformer","morphologizer","tagger","parser","trainable_lemmatizer"]
 batch_size = 512
 disabled = []
 before_creation = null
@@ -43,9 +43,6 @@ use_fast = true
 
 [components.base_transformer.model.transformer_config]
 
-[components.merge_entities]
-factory = "merge_entities"
-
 [components.morphologizer]
 factory = "morphologizer"
 extend = false

de_trf_nrp-any-py3-none-any.whl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:19d025131312ebc27af7588726406e6d54c697a0c050e2c761a776aab105c256
-size 865701680
+oid sha256:f3ceaf71530479286554f5f67226794741a6a306a84011daa39a2fa9c87709a2
+size 865702938

meta.json

@@ -1830,7 +1830,6 @@
   "pipeline":[
     "ner_transformer",
     "ner",
-    "merge_entities",
     "base_transformer",
     "morphologizer",
     "tagger",
@@ -1840,7 +1839,6 @@
   "components":[
     "ner_transformer",
     "ner",
-    "merge_entities",
     "base_transformer",
     "morphologizer",
     "tagger",

ner_transformer/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dada82ed46518721927ad87b0662e3602c24c5642e62a10a87400fa9383855d8
+oid sha256:83cbeeb733016f569fda192d17aef837b9870753c705785de7e4f3b55822f941
 size 440759145

use_custom_tokenizer.py

@@ -1,13 +1,12 @@
 from spacy.util import registry
-
-from commercial_registry_ner.spacy.custom_tokenizer.custom_tokenizer import (
-    custom_tokenizer,
-)
-
+from spacy.tokenizer import Tokenizer
+import pathlib
 
 @registry.tokenizers("customize_tokenizer")
 def make_customize_tokenizer():
     def customize_tokenizer(nlp):
-        return custom_tokenizer(nlp)
+        tokenizer = Tokenizer(nlp.vocab)
+        script_dir = pathlib.Path(__file__).parent.resolve()
+        return tokenizer.from_disk(script_dir / "tokenizer")
 
     return customize_tokenizer