the 35th line sp.spdiag(y == val ......) might be wrong. i assume that u maybe wanna write sp.spdiag([i for i in y if i == val] ......)

expect your reply,thx!

For some reason, I can't use your vectorizers in pipeline. Here is my code:

pipeline = Pipeline([ 
    ('vect', CountVectorizer(stop_words='en', ngram_range=(1,2), analyzer='word')),
    ('transform', TfIcfVectorizer(sublinear_tf=True)),
    ('clf', LinearSVC(class_weight='balanced')),
])
pipeline.fit(X, y)

And I got the following error:

D:\programs\Anaconda3\lib\site-packages\sklearn\pipeline.py in fit(self, X, y, **fit_params) 263 This estimator 264 """ --> 265 Xt, fit_params = self._fit(X, y, **fit_params) 266 if self._final_estimator is not None: 267 self._final_estimator.fit(Xt, y, **fit_params)

D:\programs\Anaconda3\lib\site-packages\sklearn\pipeline.py in _fit(self, X, y, **fit_params) 228 Xt, fitted_transformer = fit_transform_one_cached( 229 cloned_transformer, Xt, y, None, --> 230 **fit_params_steps[name]) 231 # Replace the transformer of the step with the fitted 232 # transformer. This is necessary when loading the transformer

D:\programs\Anaconda3\lib\site-packages\sklearn\externals\joblib\memory.py in call(self, *args, **kwargs) 340 341 def call(self, *args, **kwargs): --> 342 return self.func(*args, **kwargs) 343 344 def call_and_shelve(self, *args, **kwargs):

D:\programs\Anaconda3\lib\site-packages\sklearn\pipeline.py in _fit_transform_one(transformer, X, y, weight, **fit_params) 614 res = transformer.fit_transform(X, y, **fit_params) 615 else: --> 616 res = transformer.fit(X, y, **fit_params).transform(X) 617 # if we have a weight for this transformer, multiply output 618 if weight is None:

AttributeError: 'NoneType' object has no attribute 'transform'

My code works fine with TfIdfTransformer.

Python 3.7.3 sklearn 0.20.3

Fixes #20 Fixes #21

By adding sklearn.base.BaseEstimator to all vectorizers they are now capable of handling parameters from GridSearchCV.

Also, when testing I verified that they already got a "toString" for them, so two problems solved.

>>> from textvec.vectorizers import TfIcfVectorizer
>>> TfIcfVectorizer()
TfIcfVectorizer(norm=None, sublinear_tf=False)
>>> from textvec.vectorizers import TfBinIcfVectorizer
>>> TfBinIcfVectorizer()
TfBinIcfVectorizer(norm='l2', smooth_df=True, sublinear_tf=False)

I want to get text vectorization using TfIcfVectorizer() without the need to use Y label vector, Is it possible ?

tficf = vectorizers.TfIcfVectorizer() tficf_train = tficf.fit_transform(cv_train, newsgroups_train.target) # here i should give the vectorizer one argument instead of two tficf_test = tficf.transform(cv_test)

Bumps numpy from 1.14.2 to 1.22.0.

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting @dependabot rebase.

The code below throws this error:

AttributeError: 'TfBinIcfVectorizer' object has no attribute 'set_params'

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import GridSearchCV
from sklearn.svm import SVC
from sklearn.pipeline import Pipeline
from textvec.vectorizers import TfBinIcfVectorizer

pipeline = Pipeline([
    ('count', CountVectorizer()),
    ('transformer', TfBinIcfVectorizer()),
    ('model', SVC()),
])

param_grid = {
    'transformer__sublinear_tf': (True, False),
    'count__analyzer': ('word', 'char'),
    'count__max_df': (1.0,),
}

grid_search = GridSearchCV(pipeline, param_grid, n_jobs=-1, verbose=1, refit=True)
grid_search.fit(train_x, train_y)

If I replace TfBinIcfVectorizer with TfidfTransformer it does work. So it seems that there is something missing on textvec.

As shown below by the code snippet, scikit-learn's TfidfTransformer for example have a pretty representation when printed, while textvec's TfIcfVectorizer doesn't. It seems that this happens to all of then.

I'll search for way to do it like scikit-learn does.

>>> from sklearn.feature_extraction.text import TfidfTransformer
>>> TfidfTransformer()
TfidfTransformer(norm='l2', smooth_idf=True, sublinear_tf=False, use_idf=True)
>>> from textvec.vectorizers import TfIcfVectorizer
>>> TfIcfVectorizer()
<textvec.vectorizers.TfIcfVectorizer object at 0x7f88de881490>

As from this pull request it seems that TfBNS is now solved from the TODO list on README.md.

That means that it should be added to the Currently implemented methods and removed from the TODO.

Code:

...
train['title'].isnull().sum() 
# Out: 0
title_countvec = CountVectorizer(ngram_range=(1,3), max_features=300000, lowercase=True)
title_countvec.fit(train['title'], y_train)
train_title_countvec = title_countvec.transform(train['title'])
title_vectorizer = TfIcfVectorizer(norm='l2', sublinear_tf=True)
title_vectorizer.fit(train_title_countvec, y_train)
train_title_countvec = title_countvec.transform(train['title'])
np.isfinite(train_title_countvec.data).all(), np.isinf(train_title_countvec.data).any()
# Out: (True, False)
train_transformed['title'] = title_vectorizer.transform(train_title_countvec) 
# Error

Traceback:

ValueError                                Traceback (most recent call last)
<ipython-input-72-5fdf9718ecba> in <module>()
----> 1 train_transformed['title'] = title_vectorizer.transform(train_title_countvec)

/usr/local/lib/python3.5/dist-packages/textvec/vectorizers.py in transform(self, X, min_freq)
     45         X = X * sp.spdiags(self.k, 0, f, f)
     46         if self.norm:
---> 47             X = normalize(X, self.norm)
     48         return X
     49 

~/.local/lib/python3.5/site-packages/sklearn/preprocessing/data.py in normalize(X, norm, axis, copy, return_norm)
   1410 
   1411     X = check_array(X, sparse_format, copy=copy,
-> 1412                     estimator='the normalize function', dtype=FLOAT_DTYPES)
   1413     if axis == 0:
   1414         X = X.T

~/.local/lib/python3.5/site-packages/sklearn/utils/validation.py in check_array(array, accept_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, warn_on_dtype, estimator)
    429     if sp.issparse(array):
    430         array = _ensure_sparse_format(array, accept_sparse, dtype, copy,
--> 431                                       force_all_finite)
    432     else:
    433         array = np.array(array, dtype=dtype, order=order, copy=copy)

~/.local/lib/python3.5/site-packages/sklearn/utils/validation.py in _ensure_sparse_format(spmatrix, accept_sparse, dtype, copy, force_all_finite)
    304                           % spmatrix.format)
    305         else:
--> 306             _assert_all_finite(spmatrix.data)
    307     return spmatrix
    308 

~/.local/lib/python3.5/site-packages/sklearn/utils/validation.py in _assert_all_finite(X)
     42             and not np.isfinite(X).all()):
     43         raise ValueError("Input contains NaN, infinity"
---> 44                          " or a value too large for %r." % X.dtype)
     45 
     46 

ValueError: Input contains NaN, infinity or a value too large for dtype('float64').