Skip to content

Commit

Permalink
fix(grpc): pass by modelpath
Browse files Browse the repository at this point in the history
Instead of trying to derive it from the model file. In backends that
specify HF url this results in a fragile logic.

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
  • Loading branch information
mudler committed Oct 31, 2024
1 parent 48d621c commit 2c1a26e
Show file tree
Hide file tree
Showing 3 changed files with 7 additions and 6 deletions.
2 changes: 2 additions & 0 deletions backend/backend.proto
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,8 @@ message ModelOptions {

bool FlashAttention = 56;
bool NoKVOffload = 57;

string ModelPath = 59;
}

message Result {
Expand Down
10 changes: 4 additions & 6 deletions backend/python/diffusers/backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -301,13 +301,11 @@ def LoadModel(self, request, context):
self.pipe.controlnet = self.controlnet
else:
self.controlnet = None
# Assume directory from request.ModelFile.
# Only if request.LoraAdapter it's not an absolute path
if request.LoraAdapter and request.ModelFile != "" and not os.path.isabs(request.LoraAdapter) and request.LoraAdapter:
# get base path of modelFile
modelFileBase = os.path.dirname(request.ModelFile)

if request.LoraAdapter and not os.path.isabs(request.LoraAdapter):
# modify LoraAdapter to be relative to modelFileBase
request.LoraAdapter = os.path.join(modelFileBase, request.LoraAdapter)
request.LoraAdapter = os.path.join(request.ModelPath, request.LoraAdapter)

device = "cpu" if not request.CUDA else "cuda"
self.device = device
if request.LoraAdapter:
Expand Down
1 change: 1 addition & 0 deletions pkg/model/initializers.go
Original file line number Diff line number Diff line change
Expand Up @@ -425,6 +425,7 @@ func (ml *ModelLoader) grpcModel(backend string, autodetect bool, o *Options) fu
options := *o.gRPCOptions
options.Model = modelName
options.ModelFile = modelFile
options.ModelPath = ml.ModelPath

log.Debug().Msgf("GRPC: Loading model with options: %+v", options)

Expand Down

0 comments on commit 2c1a26e

Please sign in to comment.