Add Sentence Transformers integration (#7)

- Update README; modeling_gemma2.py; overwrite (979b19f0a88cf8efed7a354454d4e0b9c400df66)
- Undo weird unicode changes (8041ca86014930f5cde7da3f6614149530e85122)

README.md

@@ -1,5 +1,11 @@
 ---
 license: cc-by-nc-4.0
+pipeline_tag: feature-extraction
+tags:
+- transformers
+- sentence-transformers
+- code
+- retrieval
 ---
 <h1 align="center">Salesforce/SFR-Embedding-Code-2B_R</h1>
 
@@ -52,7 +58,7 @@ from transformers import AutoTokenizer, AutoModel
 query_instruction_example = "Given Code or Text, retrieval relevant content"
 queries = [
     "how to implement quick sort in Python?"
-    ]
+]
 
 # No instruction needed for retrieval passages
 passages = [
@@ -74,6 +80,35 @@ passage_embeddings = F.normalize(passage_embeddings, p=2, dim=1)
 
 scores = (query_embeddings @ passage_embeddings.T) * 100
 print(scores.tolist())
+# [[52.76957702636719, 26.118698120117188]]
+```
+
+#### Sentence Transformers
+
+```python
+from sentence_transformers import SentenceTransformer
+
+# Each query needs to be accompanied by an corresponding instruction describing the task.
+query_instruction_example = "Instruct: Given Code or Text, retrieval relevant content\nQuery: "
+queries = ["how to implement quick sort in Python?"]
+
+# No instruction needed for retrieval passages
+passages = [
+    "def quick_sort(arr):\n    if len(arr) <= 1:\n        return arr\n    pivot = arr[len(arr) // 2]\n    left = [x for x in arr if x < pivot]\n    middle = [x for x in arr if x == pivot]\n    right = [x for x in arr if x > pivot]\n    return quick_sort(left) + middle + quick_sort(right)",
+    "def bubble_sort(arr):\n    n = len(arr)\n    for i in range(n):\n        for j in range(0, n-i-1):\n            if arr[j] > arr[j+1]:\n                arr[j], arr[j+1] = arr[j+1], arr[j]\n    return arr"
+]
+
+# Load the Sentence Transformer model, including pooling
+model = SentenceTransformer('Salesforce/SFR-Embedding-Code-2B_R', trust_remote_code=True)
+
+# Compute the embeddings for both queries and passages. Use 'prompt' for queries only
+query_embeddings = model.encode(queries, prompt=query_instruction_example)
+passage_embeddings = model.encode(passages)
+
+# Compute the similarities between the queries and passages
+similarities = model.similarity(query_embeddings, passage_embeddings)
+print(similarities)
+# tensor([[0.5277, 0.2612]])
 ```
 
 ### Citation

modeling_gemma2.py

@@ -1350,6 +1350,9 @@ class CodeXEmbedModel2B(PreTrainedModel):
             self.tokenizer.pad_token = self.tokenizer.eos_token
         self.tokenizer.padding_side = 'right'
 
+    def forward(self, **kwargs):
+        return self.model(**kwargs)
+
     def last_token_pool(self, model_output, attention_mask):
         last_hidden_states = model_output.last_hidden_state
         sequence_lengths = attention_mask.sum(dim=1) - 1