Added a model file

Files changed (27) hide show

.gitattributes +8 -25
README.md +133 -0
config.yaml +16 -0
example.png +0 -0
pytorch_model.bin +3 -0
reproducible_research/dihard3_custom_split/development.txt +62 -0
reproducible_research/dihard3_custom_split/train.txt +192 -0
reproducible_research/expected_outputs/osd/AMI.development.rttm +0 -0
reproducible_research/expected_outputs/osd/AMI.test.rttm +0 -0
reproducible_research/expected_outputs/osd/DIHARD.development.rttm +0 -0
reproducible_research/expected_outputs/osd/DIHARD.test.rttm +0 -0
reproducible_research/expected_outputs/osd/VoxConverse.development.rttm +1614 -0
reproducible_research/expected_outputs/osd/VoxConverse.test.rttm +0 -0
reproducible_research/expected_outputs/rsg/AMI.development.rttm +0 -0
reproducible_research/expected_outputs/rsg/AMI.test.rttm +0 -0
reproducible_research/expected_outputs/rsg/DIHARD.development.rttm +0 -0
reproducible_research/expected_outputs/rsg/DIHARD.test.rttm +0 -0
reproducible_research/expected_outputs/rsg/VoxConverse.development.rttm +0 -0
reproducible_research/expected_outputs/vad/AMI.development.rttm +0 -0
reproducible_research/expected_outputs/vad/AMI.test.rttm +0 -0
reproducible_research/expected_outputs/vad/DIHARD.development.rttm +0 -0
reproducible_research/expected_outputs/vad/DIHARD.test.rttm +0 -0
reproducible_research/expected_outputs/vad/VoxConverse.development.rttm +0 -0
reproducible_research/expected_outputs/vad/VoxConverse.test.rttm +0 -0
reproducible_research/expected_outputs/vbx/AMI.rttm +0 -0
reproducible_research/expected_outputs/vbx/DIHARD.rttm +0 -0
reproducible_research/expected_outputs/vbx/VoxConverse.rttm +0 -0

.gitattributes CHANGED Viewed

@@ -1,33 +1,16 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
 *.h5 filter=lfs diff=lfs merge=lfs -text
 *.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
 *.model filter=lfs diff=lfs merge=lfs -text
 *.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
 *.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

+*.bin.* filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text
 *.h5 filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tar.gz filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
 *.joblib filter=lfs diff=lfs merge=lfs -text
 *.model filter=lfs diff=lfs merge=lfs -text
 *.msgpack filter=lfs diff=lfs merge=lfs -text
 *.pb filter=lfs diff=lfs merge=lfs -text
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text

README.md CHANGED Viewed

@@ -1,3 +1,136 @@
 ---
 license: mit
 ---

 ---
+tags:
+- pyannote
+- pyannote-audio
+- pyannote-audio-model
+- audio
+- voice
+- speech
+- speaker
+- speaker-segmentation
+- voice-activity-detection
+- overlapped-speech-detection
+- resegmentation
+datasets:
+- ami
+- dihard
+- voxconverse
 license: mit
+inference: false
 ---
+# 🎹 Speaker segmentation
+![Example](example.png)
+Model from *[End-to-end speaker segmentation for overlap-aware resegmentation](http://arxiv.org/abs/2104.04045)*,
+by Hervé Bredin and Antoine Laurent.
+[Online demo](https://huggingface.co/spaces/pyannote/pretrained-pipelines) is available as a Hugging Face Space.
+## Support
+For commercial enquiries and scientific consulting, please contact [me](mailto:[email protected]).
+For [technical questions](https://github.com/pyannote/pyannote-audio/discussions) and [bug reports](https://github.com/pyannote/pyannote-audio/issues), please check [pyannote.audio](https://github.com/pyannote/pyannote-audio) Github repository.
+## Usage
+Relies on pyannote.audio 2.0 currently in development: see [installation instructions](https://github.com/pyannote/pyannote-audio/tree/develop#installation).
+### Voice activity detection
+```python
+from pyannote.audio.pipelines import VoiceActivityDetection
+pipeline = VoiceActivityDetection(segmentation="pyannote/segmentation")
+HYPER_PARAMETERS = {
+  # onset/offset activation thresholds
+  "onset": 0.5, "offset": 0.5,
+  # remove speech regions shorter than that many seconds.
+  "min_duration_on": 0.0,
+  # fill non-speech regions shorter than that many seconds.
+  "min_duration_off": 0.0
+}
+pipeline.instantiate(HYPER_PARAMETERS)
+vad = pipeline("audio.wav")
+# `vad` is a pyannote.core.Annotation instance containing speech regions
+```
+### Overlapped speech detection
+```python
+from pyannote.audio.pipelines import OverlappedSpeechDetection
+pipeline = OverlappedSpeechDetection(segmentation="pyannote/segmentation")
+pipeline.instantiate(HYPER_PARAMETERS)
+osd = pipeline("audio.wav")
+# `osd` is a pyannote.core.Annotation instance containing overlapped speech regions
+```
+### Resegmentation
+```python
+from pyannote.audio.pipelines import Resegmentation
+pipeline = Resegmentation(segmentation="pyannote/segmentation",
+                          diarization="baseline")
+pipeline.instantiate(HYPER_PARAMETERS)
+resegmented_baseline = pipeline({"audio": "audio.wav", "baseline": baseline})
+# where `baseline` should be provided as a pyannote.core.Annotation instance
+```
+### Raw scores
+```python
+from pyannote.audio import Inference
+inference = Inference("pyannote/segmentation")
+segmentation = inference("audio.wav")
+# `segmentation` is a pyannote.core.SlidingWindowFeature
+# instance containing raw segmentation scores like the
+# one pictured above (output)
+```
+## Reproducible research
+In order to reproduce the results of the paper ["End-to-end speaker segmentation for overlap-aware resegmentation
+"](https://arxiv.org/abs/2104.04045), use `pyannote/segmentation@Interspeech2021` with the following hyper-parameters:
+| Voice activity detection | `onset` | `offset` | `min_duration_on` | `min_duration_off` |
+| ------------------------ | ------- | -------- | ----------------- | ------------------ |
+| AMI Mix-Headset          | 0.684   | 0.577    | 0.181             | 0.037              |
+| DIHARD3                  | 0.767   | 0.377    | 0.136             | 0.067              |
+| VoxConverse              | 0.767   | 0.713    | 0.182             | 0.501              |
+| Overlapped speech detection | `onset` | `offset` | `min_duration_on` | `min_duration_off` |
+| --------------------------- | ------- | -------- | ----------------- | ------------------ |
+| AMI Mix-Headset             | 0.448   | 0.362    | 0.116             | 0.187              |
+| DIHARD3                     | 0.430   | 0.320    | 0.091             | 0.144              |
+| VoxConverse                 | 0.587   | 0.426    | 0.337             | 0.112              |
+| Resegmentation of VBx | `onset` | `offset` | `min_duration_on` | `min_duration_off` |
+| --------------------- | ------- | -------- | ----------------- | ------------------ |
+| AMI Mix-Headset       | 0.542   | 0.527    | 0.044             | 0.705              |
+| DIHARD3               | 0.592   | 0.489    | 0.163             | 0.182              |
+| VoxConverse           | 0.537   | 0.724    | 0.410             | 0.563              |
+Expected outputs (and VBx baseline) are also provided in the `/reproducible_research` sub-directories.
+## Citation
+```bibtex
+@inproceedings{Bredin2021,
+  Title = {{End-to-end speaker segmentation for overlap-aware resegmentation}},
+  Author = {{Bredin}, Herv{\'e} and {Laurent}, Antoine},
+  Booktitle = {Proc. Interspeech 2021},
+  Address = {Brno, Czech Republic},
+  Month = {August},
+  Year = {2021},
+```
+```bibtex
+@inproceedings{Bredin2020,
+  Title = {{pyannote.audio: neural building blocks for speaker diarization}},
+  Author = {{Bredin}, Herv{\'e} and {Yin}, Ruiqing and {Coria}, Juan Manuel and {Gelly}, Gregory and {Korshunov}, Pavel and {Lavechin}, Marvin and {Fustes}, Diego and {Titeux}, Hadrien and {Bouaziz}, Wassim and {Gill}, Marie-Philippe},
+  Booktitle = {ICASSP 2020, IEEE International Conference on Acoustics, Speech, and Signal Processing},
+  Address = {Barcelona, Spain},
+  Month = {May},
+  Year = {2020},
+}
+```

config.yaml ADDED Viewed

	@@ -0,0 +1,16 @@

+task:
+  _target_: pyannote.audio.tasks.Segmentation
+  duration: 5.0
+  max_num_speakers: 3
+model:
+  _target_: pyannote.audio.models.segmentation.PyanNet
+  sincnet:
+    stride: 10
+  lstm:
+    hidden_size: 128
+    num_layers: 4
+    bidirectional: true
+    monolithic: true
+  linear:
+    hidden_size: 128
+    num_layers: 2

example.png ADDED Viewed

pytorch_model.bin ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:0b5b3216d60a2d32fc086b47ea8c67589aaeb26b7e07fcbe620d6d0b83e209ea
+size 17719103

reproducible_research/dihard3_custom_split/development.txt ADDED Viewed

	@@ -0,0 +1,62 @@

+DH_DEV_0001
+DH_DEV_0023
+DH_DEV_0071
+DH_DEV_0078
+DH_DEV_0002
+DH_DEV_0015
+DH_DEV_0039
+DH_DEV_0003
+DH_DEV_0004
+DH_DEV_0006
+DH_DEV_0010
+DH_DEV_0012
+DH_DEV_0016
+DH_DEV_0024
+DH_DEV_0028
+DH_DEV_0033
+DH_DEV_0041
+DH_DEV_0042
+DH_DEV_0044
+DH_DEV_0047
+DH_DEV_0048
+DH_DEV_0056
+DH_DEV_0005
+DH_DEV_0008
+DH_DEV_0009
+DH_DEV_0017
+DH_DEV_0018
+DH_DEV_0007
+DH_DEV_0030
+DH_DEV_0037
+DH_DEV_0011
+DH_DEV_0027
+DH_DEV_0062
+DH_DEV_0013
+DH_DEV_0022
+DH_DEV_0025
+DH_DEV_0029
+DH_DEV_0031
+DH_DEV_0038
+DH_DEV_0046
+DH_DEV_0058
+DH_DEV_0014
+DH_DEV_0021
+DH_DEV_0026
+DH_DEV_0036
+DH_DEV_0043
+DH_DEV_0049
+DH_DEV_0055
+DH_DEV_0060
+DH_DEV_0068
+DH_DEV_0076
+DH_DEV_0077
+DH_DEV_0090
+DH_DEV_0032
+DH_DEV_0035
+DH_DEV_0065
+DH_DEV_0034
+DH_DEV_0123
+DH_DEV_0133
+DH_DEV_0051
+DH_DEV_0052
+DH_DEV_0054

reproducible_research/dihard3_custom_split/train.txt ADDED Viewed

	@@ -0,0 +1,192 @@

+DH_DEV_0080
+DH_DEV_0102
+DH_DEV_0122
+DH_DEV_0144
+DH_DEV_0145
+DH_DEV_0173
+DH_DEV_0176
+DH_DEV_0195
+DH_DEV_0214
+DH_DEV_0221
+DH_DEV_0225
+DH_DEV_0246
+DH_DEV_0066
+DH_DEV_0079
+DH_DEV_0103
+DH_DEV_0113
+DH_DEV_0124
+DH_DEV_0141
+DH_DEV_0206
+DH_DEV_0216
+DH_DEV_0230
+DH_DEV_0059
+DH_DEV_0063
+DH_DEV_0064
+DH_DEV_0070
+DH_DEV_0072
+DH_DEV_0074
+DH_DEV_0075
+DH_DEV_0081
+DH_DEV_0082
+DH_DEV_0084
+DH_DEV_0091
+DH_DEV_0094
+DH_DEV_0098
+DH_DEV_0100
+DH_DEV_0106
+DH_DEV_0109
+DH_DEV_0114
+DH_DEV_0116
+DH_DEV_0126
+DH_DEV_0127
+DH_DEV_0128
+DH_DEV_0130
+DH_DEV_0135
+DH_DEV_0137
+DH_DEV_0138
+DH_DEV_0142
+DH_DEV_0143
+DH_DEV_0156
+DH_DEV_0166
+DH_DEV_0174
+DH_DEV_0178
+DH_DEV_0186
+DH_DEV_0191
+DH_DEV_0196
+DH_DEV_0209
+DH_DEV_0223
+DH_DEV_0234
+DH_DEV_0235
+DH_DEV_0237
+DH_DEV_0238
+DH_DEV_0239
+DH_DEV_0240
+DH_DEV_0241
+DH_DEV_0248
+DH_DEV_0251
+DH_DEV_0252
+DH_DEV_0019
+DH_DEV_0020
+DH_DEV_0045
+DH_DEV_0050
+DH_DEV_0053
+DH_DEV_0073
+DH_DEV_0087
+DH_DEV_0089
+DH_DEV_0096
+DH_DEV_0132
+DH_DEV_0139
+DH_DEV_0167
+DH_DEV_0177
+DH_DEV_0184
+DH_DEV_0201
+DH_DEV_0210
+DH_DEV_0228
+DH_DEV_0236
+DH_DEV_0040
+DH_DEV_0057
+DH_DEV_0105
+DH_DEV_0108
+DH_DEV_0163
+DH_DEV_0188
+DH_DEV_0193
+DH_DEV_0204
+DH_DEV_0208
+DH_DEV_0212
+DH_DEV_0217
+DH_DEV_0093
+DH_DEV_0146
+DH_DEV_0154
+DH_DEV_0155
+DH_DEV_0158
+DH_DEV_0183
+DH_DEV_0187
+DH_DEV_0229
+DH_DEV_0245
+DH_DEV_0061
+DH_DEV_0067
+DH_DEV_0069
+DH_DEV_0083
+DH_DEV_0086
+DH_DEV_0088
+DH_DEV_0118
+DH_DEV_0121
+DH_DEV_0147
+DH_DEV_0153
+DH_DEV_0160
+DH_DEV_0164
+DH_DEV_0165
+DH_DEV_0168
+DH_DEV_0172
+DH_DEV_0181
+DH_DEV_0192
+DH_DEV_0194
+DH_DEV_0202
+DH_DEV_0203
+DH_DEV_0218
+DH_DEV_0250
+DH_DEV_0253
+DH_DEV_0254
+DH_DEV_0095
+DH_DEV_0097
+DH_DEV_0099
+DH_DEV_0104
+DH_DEV_0107
+DH_DEV_0110
+DH_DEV_0111
+DH_DEV_0115
+DH_DEV_0117
+DH_DEV_0129
+DH_DEV_0148
+DH_DEV_0149
+DH_DEV_0150
+DH_DEV_0151
+DH_DEV_0152
+DH_DEV_0157
+DH_DEV_0161
+DH_DEV_0171
+DH_DEV_0182
+DH_DEV_0185
+DH_DEV_0190
+DH_DEV_0197
+DH_DEV_0198
+DH_DEV_0200
+DH_DEV_0205
+DH_DEV_0213
+DH_DEV_0215
+DH_DEV_0219
+DH_DEV_0220
+DH_DEV_0222
+DH_DEV_0224
+DH_DEV_0231
+DH_DEV_0242
+DH_DEV_0243
+DH_DEV_0247
+DH_DEV_0249
+DH_DEV_0092
+DH_DEV_0112
+DH_DEV_0119
+DH_DEV_0131
+DH_DEV_0136
+DH_DEV_0175
+DH_DEV_0199
+DH_DEV_0207
+DH_DEV_0232
+DH_DEV_0134
+DH_DEV_0159
+DH_DEV_0169
+DH_DEV_0170
+DH_DEV_0180
+DH_DEV_0189
+DH_DEV_0211
+DH_DEV_0226
+DH_DEV_0244
+DH_DEV_0085
+DH_DEV_0101
+DH_DEV_0120
+DH_DEV_0125
+DH_DEV_0140
+DH_DEV_0162
+DH_DEV_0179
+DH_DEV_0227
+DH_DEV_0233

reproducible_research/expected_outputs/osd/AMI.development.rttm ADDED Viewed