Skip to content

Commit

Permalink
Remove dead code
Browse files Browse the repository at this point in the history
  • Loading branch information
mattdangerw committed Jun 4, 2024
1 parent 38c6608 commit ba9a30c
Show file tree
Hide file tree
Showing 41 changed files with 77 additions and 175 deletions.
4 changes: 2 additions & 2 deletions keras_nlp/src/models/albert/albert_masked_lm_preprocessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,11 @@
from absl import logging

from keras_nlp.src.api_export import keras_nlp_export
from keras_nlp.src.backend import keras
from keras_nlp.src.layers.preprocessing.masked_lm_mask_generator import (
MaskedLMMaskGenerator,
)
from keras_nlp.src.models.albert.albert_preprocessor import AlbertPreprocessor
from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight


@keras_nlp_export("keras_nlp.models.AlbertMaskedLMPreprocessor")
Expand Down Expand Up @@ -191,4 +191,4 @@ def call(self, x, y=None, sample_weight=None):
}
y = masker_outputs["mask_ids"]
sample_weight = masker_outputs["mask_weights"]
return pack_x_y_sample_weight(x, y, sample_weight)
return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
4 changes: 2 additions & 2 deletions keras_nlp/src/models/albert/albert_preprocessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
# limitations under the License.

from keras_nlp.src.api_export import keras_nlp_export
from keras_nlp.src.backend import keras
from keras_nlp.src.layers.preprocessing.multi_segment_packer import (
MultiSegmentPacker,
)
Expand All @@ -21,7 +22,6 @@
from keras_nlp.src.utils.keras_utils import (
convert_inputs_to_list_of_tensor_segments,
)
from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight


@keras_nlp_export("keras_nlp.models.AlbertPreprocessor")
Expand Down Expand Up @@ -191,7 +191,7 @@ def call(self, x, y=None, sample_weight=None):
"segment_ids": segment_ids,
"padding_mask": token_ids != self.tokenizer.pad_token_id,
}
return pack_x_y_sample_weight(x, y, sample_weight)
return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)

@property
def sequence_length(self):
Expand Down
4 changes: 2 additions & 2 deletions keras_nlp/src/models/bart/bart_preprocessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,13 @@


from keras_nlp.src.api_export import keras_nlp_export
from keras_nlp.src.backend import keras
from keras_nlp.src.layers.preprocessing.start_end_packer import StartEndPacker
from keras_nlp.src.models.bart.bart_tokenizer import BartTokenizer
from keras_nlp.src.models.preprocessor import Preprocessor
from keras_nlp.src.utils.keras_utils import (
convert_inputs_to_list_of_tensor_segments,
)
from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight


@keras_nlp_export("keras_nlp.models.BartPreprocessor")
Expand Down Expand Up @@ -231,7 +231,7 @@ def call(
"decoder_padding_mask": decoder_padding_mask,
}

return pack_x_y_sample_weight(x, y, sample_weight)
return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)

def get_config(self):
config = super().get_config()
Expand Down
4 changes: 2 additions & 2 deletions keras_nlp/src/models/bart/bart_seq_2_seq_lm_preprocessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,12 @@
from absl import logging

from keras_nlp.src.api_export import keras_nlp_export
from keras_nlp.src.backend import keras
from keras_nlp.src.backend import ops
from keras_nlp.src.models.bart.bart_preprocessor import BartPreprocessor
from keras_nlp.src.utils.keras_utils import (
convert_inputs_to_list_of_tensor_segments,
)
from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight


@keras_nlp_export("keras_nlp.models.BartSeq2SeqLMPreprocessor")
Expand Down Expand Up @@ -170,7 +170,7 @@ def call(
# left (i.e., the next token).
y = decoder_token_ids[..., 1:]
sample_weight = decoder_padding_mask[..., 1:]
return pack_x_y_sample_weight(x, y, sample_weight)
return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)

def generate_preprocess(
self,
Expand Down
4 changes: 2 additions & 2 deletions keras_nlp/src/models/bert/bert_masked_lm_preprocessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,11 @@
from absl import logging

from keras_nlp.src.api_export import keras_nlp_export
from keras_nlp.src.backend import keras
from keras_nlp.src.layers.preprocessing.masked_lm_mask_generator import (
MaskedLMMaskGenerator,
)
from keras_nlp.src.models.bert.bert_preprocessor import BertPreprocessor
from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight


@keras_nlp_export("keras_nlp.models.BertMaskedLMPreprocessor")
Expand Down Expand Up @@ -183,7 +183,7 @@ def call(self, x, y=None, sample_weight=None):
}
y = masker_outputs["mask_ids"]
sample_weight = masker_outputs["mask_weights"]
return pack_x_y_sample_weight(x, y, sample_weight)
return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)

def get_config(self):
config = super().get_config()
Expand Down
4 changes: 2 additions & 2 deletions keras_nlp/src/models/bert/bert_preprocessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
# limitations under the License.

from keras_nlp.src.api_export import keras_nlp_export
from keras_nlp.src.backend import keras
from keras_nlp.src.layers.preprocessing.multi_segment_packer import (
MultiSegmentPacker,
)
Expand All @@ -21,7 +22,6 @@
from keras_nlp.src.utils.keras_utils import (
convert_inputs_to_list_of_tensor_segments,
)
from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight


@keras_nlp_export("keras_nlp.models.BertPreprocessor")
Expand Down Expand Up @@ -159,7 +159,7 @@ def call(self, x, y=None, sample_weight=None):
"segment_ids": segment_ids,
"padding_mask": token_ids != self.tokenizer.pad_token_id,
}
return pack_x_y_sample_weight(x, y, sample_weight)
return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)

def get_config(self):
config = super().get_config()
Expand Down
4 changes: 2 additions & 2 deletions keras_nlp/src/models/bloom/bloom_causal_lm_preprocessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,12 @@
from absl import logging

from keras_nlp.src.api_export import keras_nlp_export
from keras_nlp.src.backend import keras
from keras_nlp.src.backend import ops
from keras_nlp.src.models.bloom.bloom_preprocessor import BloomPreprocessor
from keras_nlp.src.utils.keras_utils import (
convert_inputs_to_list_of_tensor_segments,
)
from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight


@keras_nlp_export("keras_nlp.models.BloomCausalLMPreprocessor")
Expand Down Expand Up @@ -130,7 +130,7 @@ def call(
}
# Target `y` will be the next token.
y, sample_weight = token_ids[..., 1:], padding_mask[..., 1:]
return pack_x_y_sample_weight(x, y, sample_weight)
return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)

def generate_preprocess(
self,
Expand Down
4 changes: 2 additions & 2 deletions keras_nlp/src/models/bloom/bloom_preprocessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,13 @@


from keras_nlp.src.api_export import keras_nlp_export
from keras_nlp.src.backend import keras
from keras_nlp.src.layers.preprocessing.start_end_packer import StartEndPacker
from keras_nlp.src.models.bloom.bloom_tokenizer import BloomTokenizer
from keras_nlp.src.models.preprocessor import Preprocessor
from keras_nlp.src.utils.keras_utils import (
convert_inputs_to_list_of_tensor_segments,
)
from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight


@keras_nlp_export("keras_nlp.models.BloomPreprocessor")
Expand Down Expand Up @@ -159,7 +159,7 @@ def call(
"token_ids": token_ids,
"padding_mask": padding_mask,
}
return pack_x_y_sample_weight(x, y, sample_weight)
return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)

def get_config(self):
config = super().get_config()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,13 @@
from absl import logging

from keras_nlp.src.api_export import keras_nlp_export
from keras_nlp.src.backend import keras
from keras_nlp.src.layers.preprocessing.masked_lm_mask_generator import (
MaskedLMMaskGenerator,
)
from keras_nlp.src.models.deberta_v3.deberta_v3_preprocessor import (
DebertaV3Preprocessor,
)
from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight


@keras_nlp_export("keras_nlp.models.DebertaV3MaskedLMPreprocessor")
Expand Down Expand Up @@ -188,4 +188,4 @@ def call(self, x, y=None, sample_weight=None):
}
y = masker_outputs["mask_ids"]
sample_weight = masker_outputs["mask_weights"]
return pack_x_y_sample_weight(x, y, sample_weight)
return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
4 changes: 2 additions & 2 deletions keras_nlp/src/models/deberta_v3/deberta_v3_preprocessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@


from keras_nlp.src.api_export import keras_nlp_export
from keras_nlp.src.backend import keras
from keras_nlp.src.layers.preprocessing.multi_segment_packer import (
MultiSegmentPacker,
)
Expand All @@ -24,7 +25,6 @@
from keras_nlp.src.utils.keras_utils import (
convert_inputs_to_list_of_tensor_segments,
)
from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight


@keras_nlp_export("keras_nlp.models.DebertaV3Preprocessor")
Expand Down Expand Up @@ -191,7 +191,7 @@ def call(self, x, y=None, sample_weight=None):
"token_ids": token_ids,
"padding_mask": token_ids != self.tokenizer.pad_token_id,
}
return pack_x_y_sample_weight(x, y, sample_weight)
return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)

@property
def sequence_length(self):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,13 @@
from absl import logging

from keras_nlp.src.api_export import keras_nlp_export
from keras_nlp.src.backend import keras
from keras_nlp.src.layers.preprocessing.masked_lm_mask_generator import (
MaskedLMMaskGenerator,
)
from keras_nlp.src.models.distil_bert.distil_bert_preprocessor import (
DistilBertPreprocessor,
)
from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight


@keras_nlp_export("keras_nlp.models.DistilBertMaskedLMPreprocessor")
Expand Down Expand Up @@ -179,7 +179,7 @@ def call(self, x, y=None, sample_weight=None):
}
y = masker_outputs["mask_ids"]
sample_weight = masker_outputs["mask_weights"]
return pack_x_y_sample_weight(x, y, sample_weight)
return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)

def get_config(self):
config = super().get_config()
Expand Down
4 changes: 2 additions & 2 deletions keras_nlp/src/models/distil_bert/distil_bert_preprocessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@


from keras_nlp.src.api_export import keras_nlp_export
from keras_nlp.src.backend import keras
from keras_nlp.src.layers.preprocessing.multi_segment_packer import (
MultiSegmentPacker,
)
Expand All @@ -24,7 +25,6 @@
from keras_nlp.src.utils.keras_utils import (
convert_inputs_to_list_of_tensor_segments,
)
from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight


@keras_nlp_export("keras_nlp.models.DistilBertPreprocessor")
Expand Down Expand Up @@ -150,7 +150,7 @@ def call(self, x, y=None, sample_weight=None):
"token_ids": token_ids,
"padding_mask": token_ids != self.tokenizer.pad_token_id,
}
return pack_x_y_sample_weight(x, y, sample_weight)
return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)

def get_config(self):
config = super().get_config()
Expand Down
4 changes: 2 additions & 2 deletions keras_nlp/src/models/electra/electra_preprocessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
# limitations under the License.

from keras_nlp.src.api_export import keras_nlp_export
from keras_nlp.src.backend import keras
from keras_nlp.src.layers.preprocessing.multi_segment_packer import (
MultiSegmentPacker,
)
Expand All @@ -21,7 +22,6 @@
from keras_nlp.src.utils.keras_utils import (
convert_inputs_to_list_of_tensor_segments,
)
from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight


@keras_nlp_export("keras_nlp.models.ElectraPreprocessor")
Expand Down Expand Up @@ -150,4 +150,4 @@ def call(self, x, y=None, sample_weight=None):
"segment_ids": segment_ids,
"padding_mask": token_ids != self.tokenizer.pad_token_id,
}
return pack_x_y_sample_weight(x, y, sample_weight)
return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
4 changes: 2 additions & 2 deletions keras_nlp/src/models/f_net/f_net_masked_lm_preprocessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,11 @@
from absl import logging

from keras_nlp.src.api_export import keras_nlp_export
from keras_nlp.src.backend import keras
from keras_nlp.src.layers.preprocessing.masked_lm_mask_generator import (
MaskedLMMaskGenerator,
)
from keras_nlp.src.models.f_net.f_net_preprocessor import FNetPreprocessor
from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight


@keras_nlp_export("keras_nlp.models.FNetMaskedLMPreprocessor")
Expand Down Expand Up @@ -193,4 +193,4 @@ def call(self, x, y=None, sample_weight=None):
}
y = masker_outputs["mask_ids"]
sample_weight = masker_outputs["mask_weights"]
return pack_x_y_sample_weight(x, y, sample_weight)
return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
4 changes: 2 additions & 2 deletions keras_nlp/src/models/f_net/f_net_preprocessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@


from keras_nlp.src.api_export import keras_nlp_export
from keras_nlp.src.backend import keras
from keras_nlp.src.layers.preprocessing.multi_segment_packer import (
MultiSegmentPacker,
)
Expand All @@ -22,7 +23,6 @@
from keras_nlp.src.utils.keras_utils import (
convert_inputs_to_list_of_tensor_segments,
)
from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight


@keras_nlp_export("keras_nlp.models.FNetPreprocessor")
Expand Down Expand Up @@ -162,7 +162,7 @@ def call(self, x, y=None, sample_weight=None):
"token_ids": token_ids,
"segment_ids": segment_ids,
}
return pack_x_y_sample_weight(x, y, sample_weight)
return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)

@property
def sequence_length(self):
Expand Down
4 changes: 2 additions & 2 deletions keras_nlp/src/models/falcon/falcon_causal_lm_preprocessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,12 @@
from absl import logging

from keras_nlp.src.api_export import keras_nlp_export
from keras_nlp.src.backend import keras
from keras_nlp.src.backend import ops
from keras_nlp.src.models.falcon.falcon_preprocessor import FalconPreprocessor
from keras_nlp.src.utils.keras_utils import (
convert_inputs_to_list_of_tensor_segments,
)
from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight


@keras_nlp_export("keras_nlp.models.FalconCausalLMPreprocessor")
Expand Down Expand Up @@ -130,7 +130,7 @@ def call(
}
# Target `y` will be the next token.
y, sample_weight = token_ids[..., 1:], padding_mask[..., 1:]
return pack_x_y_sample_weight(x, y, sample_weight)
return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)

def generate_preprocess(
self,
Expand Down
4 changes: 2 additions & 2 deletions keras_nlp/src/models/falcon/falcon_preprocessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,13 @@


from keras_nlp.src.api_export import keras_nlp_export
from keras_nlp.src.backend import keras
from keras_nlp.src.layers.preprocessing.start_end_packer import StartEndPacker
from keras_nlp.src.models.falcon.falcon_tokenizer import FalconTokenizer
from keras_nlp.src.models.preprocessor import Preprocessor
from keras_nlp.src.utils.keras_utils import (
convert_inputs_to_list_of_tensor_segments,
)
from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight


@keras_nlp_export("keras_nlp.models.FalconPreprocessor")
Expand Down Expand Up @@ -161,7 +161,7 @@ def call(
"token_ids": token_ids,
"padding_mask": padding_mask,
}
return pack_x_y_sample_weight(x, y, sample_weight)
return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)

def get_config(self):
config = super().get_config()
Expand Down
Loading

0 comments on commit ba9a30c

Please sign in to comment.