Upload EncT5ForSequenceClassification

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1e9cc0194fa5bfc256b2e2d47affe664f166cdaf29430947220e1606223691cc
+oid sha256:e67a80a5bd78ab3885be58d157623db44c1ff78204817af457bd31b00e6b49aa
 size 476301088

modeling_enct5.py

@@ -93,6 +93,7 @@ class EncT5ForSequenceClassification(EncT5PreTrainedModel):
 
         # Initiate decoder embedding from scratch and define the corresponding latent vector vocabulary size.
         self.decoder_embeddings = nn.Embedding(config.decoder_vocab_size, config.d_model)
+        self.transformer.get_decoder().set_input_embeddings(self.decoder_embeddings)
 
         # Initiate decoder projection head from scratch.
         if config.problem_type == "multi_label_classification":
@@ -107,14 +108,18 @@ class EncT5ForSequenceClassification(EncT5PreTrainedModel):
 
     def load_weights_from_pretrained_t5(self, model_path: str):
         pretrained_t5_model = T5Model.from_pretrained(model_path)
-        self.transformer.load_state_dict(pretrained_t5_model.state_dict(), strict=False)
+
+        # Override the decoder embedding weights to make them the correct shape.
+        pretrained_state_dict = pretrained_t5_model.state_dict()
+        pretrained_state_dict["decoder.embed_tokens.weight"] = self.decoder_embeddings.state_dict()["weight"]
+
+        self.transformer.load_state_dict(pretrained_state_dict, strict=False)
 
     def prepare_for_fine_tuning(self):
         r"""
         Prepares the model for fine-tuning by re-initializing the necessary weights for fine-tuning. This step should be
         performed after loading the pre-trained T5 model but before fine-tuning.
         """
-        self.transformer.get_decoder().set_input_embeddings(self.decoder_embeddings)
         self.transformer.get_decoder().apply(self._init_weights)
         self._init_weights(self.classification_head)