Skip to content

feat: Improve loading, support more formats #34

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Pringled merged 24 commits into from
May 12, 2026
Merged

feat: Improve loading, support more formats #34

Changes from 1 commit
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
1bd975a
Added flexible loading with from_pretrained()
Pringled Mar 13, 2026
41b2b78
Update tests
Pringled Mar 13, 2026
fd6f6e1
Updated tests and loading logic
Pringled Mar 13, 2026
62df8c2
Refactoring logic
Pringled Mar 13, 2026
1570b25
Update tests
Pringled Mar 13, 2026
dc79154
Changed unk token logic
Pringled Mar 13, 2026
ab4c7f4
Cleaning up code
Pringled Mar 13, 2026
31859f8
Cleaning up code
Pringled Mar 13, 2026
554b7df
Cleaning up code
Pringled Mar 13, 2026
c27d7b6
Refactoring logic
Pringled Mar 13, 2026
72d61ab
Simplified code
Pringled Mar 21, 2026
002f697
Simplified code
Pringled Mar 21, 2026
039d7f3
Ensure Python parity
Pringled Mar 22, 2026
19ccc04
Resolve conflicts
Pringled Mar 22, 2026
123a06a
Fixes
Pringled Mar 22, 2026
0c9cd70
Updated comments
Pringled Mar 22, 2026
9e02042
Simplified code
Pringled Mar 22, 2026
6bbba18
Merge origin/main into pr-34
Pringled Apr 10, 2026
bbabada
test: generate model layout fixtures
Pringled Apr 10, 2026
97ae3c0
Improve code quality
Pringled May 12, 2026
aa6edbe
Improve code quality
Pringled May 12, 2026
ce77799
Bump version
Pringled May 12, 2026
cdc9f72
Update docs
Pringled May 12, 2026
a8b592b
Update docs
Pringled May 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Cleaning up code
  • Loading branch information
@Pringled
Pringled committed Mar 13, 2026
commit 31859f8c40c4372497d95cef9a78f6dc8ec6820e
12 changes: 6 additions & 6 deletions src/model.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ struct ResolvedPaths {
layout: ModelLayout,
}

struct Layout {
struct LayoutSpec {
config_file: &'static str,
tokenizer_file: &'static str,
model_file: &'static str,
Expand All @@ -45,9 +45,9 @@ struct Layout {
layout: ModelLayout,
}

static LAYOUTS: &[Layout] = &[
static LAYOUTS: &[LayoutSpec] = &[
// 1. Native model2vec
Layout {
LayoutSpec {
config_file: "config.json",
tokenizer_file: "tokenizer.json",
model_file: "model.safetensors",
Expand All @@ -56,7 +56,7 @@ static LAYOUTS: &[Layout] = &[
layout: ModelLayout::Native,
},
// 2. Sentence Transformers root layout
Layout {
LayoutSpec {
config_file: "config_sentence_transformers.json",
tokenizer_file: "tokenizer.json",
model_file: "model.safetensors",
Expand All @@ -65,7 +65,7 @@ static LAYOUTS: &[Layout] = &[
layout: ModelLayout::SentenceTransformers,
},
// 3. Sentence Transformers 0_StaticEmbedding subfolder
Layout {
LayoutSpec {
config_file: "config_sentence_transformers.json",
tokenizer_file: "0_StaticEmbedding/tokenizer.json",
model_file: "0_StaticEmbedding/model.safetensors",
Expand All @@ -74,7 +74,7 @@ static LAYOUTS: &[Layout] = &[
layout: ModelLayout::SentenceTransformers,
},
// 4. Config-in-parent (caller passed subfolder pointing directly at model files)
Layout {
LayoutSpec {
config_file: "config_sentence_transformers.json",
tokenizer_file: "tokenizer.json",
model_file: "model.safetensors",
Expand Down