问题

I have a pandas data frame which has a datetime index which looks like this:

df =

           Fruit    Quantity
01/02/10    Apple   4
01/02/10    Apple   6
01/02/10    Pear    7
01/02/10    Grape   8
01/02/10    Grape   5
02/02/10    Apple   2
02/02/10    Fruit   6
02/02/10    Pear    8
02/02/10    Pear    5

Now for each date and for each fruit I only want one value (preferably the top one) and the rest of the fruit for the date to remain zero. So desired output is as follows:

           Fruit    Quantity
01/02/10    Apple   4
01/02/10    Apple   0
01/02/10    Pear    7
01/02/10    Grape   8
01/02/10    Grape   0
02/02/10    Apple   2
02/02/10    Fruit   6
02/02/10    Pear    8
02/02/10    Pear    0

This is only a small example, but my main data frame has over 3 million rows and the fruit are not necessarily in order per date.

Thanks

回答1:

You can use :

set_index for index name
groupby with cumcount for Counter per groups
get boolean mask for first groups by eq
set 0 by where and mask

m = df.rename_axis('Date').groupby(['Date', 'Fruit']).cumcount().eq(0)
df['Quantity'] = df['Quantity'].where(m, 0)
print (df)
          Fruit  Quantity
01/02/10  Apple         4
01/02/10  Apple         0
01/02/10   Pear         7
01/02/10  Grape         8
01/02/10  Grape         0
02/02/10  Apple         2
02/02/10  Fruit         6
02/02/10   Pear         8
02/02/10   Pear         0

Another solution with reset_index, but is necessary convert boolen mask to numpy array by values, because different indices:

m = df.reset_index().groupby(['index', 'Fruit']).cumcount().eq(0)
df['Quantity'] = df['Quantity'].where(m.values, 0)
print (df)
          Fruit  Quantity
01/02/10  Apple         4
01/02/10  Apple         0
01/02/10   Pear         7
01/02/10  Grape         8
01/02/10  Grape         0
02/02/10  Apple         2
02/02/10  Fruit         6
02/02/10   Pear         8
02/02/10   Pear         0

Timings:

np.random.seed(1235)

N = 10000
L = ['Apple','Pear','Grape','Fruit']
idx = np.repeat(pd.date_range('2017-010-01', periods=N/20).strftime('%d/%m/%y'), 20)
df = (pd.DataFrame({'Fruit': np.random.choice(L, N),
                   'Quantity':np.random.randint(100, size=N), 'idx':idx})
      .sort_values(['Fruit','idx'])
      .set_index('idx')
      .rename_axis(None))             

#print (df)

def jez1(df):
    m = df.rename_axis('Date').groupby(['Date', 'Fruit']).cumcount().eq(0)
    df['Quantity'] = df['Quantity'].where(m, 0)
    return df

def jez2(df):
    m = df.reset_index().groupby(['index', 'Fruit']).cumcount().eq(0)
    df['Quantity'] = df['Quantity'].where(m.values, 0)
    return df

def rnso(df):
    df['date_fruit'] = df.index+df.Fruit # new column with date and fruit merged
    dflist = pd.unique(df.date_fruit)    # find its unique values
    dfv = df.values                      # get rows as list of lists
    for i in dflist:                     # for each unique date-fruit combination
        done = False
        for c in range(len(dfv)): 
            if dfv[c][2] == i:           # check each row
                if done: 
                    dfv[c][1] = 0        # if not first, make quantity as 0
                else: 
                    done = True

    # create new dataframe with new data: 
    newdf = pd.DataFrame(data=dfv, columns=df.columns, index=df.index)
    return newdf.iloc[:,:2]

print (jez1(df))      
print (jez2(df))      
print (rnso(df))      

In [189]: %timeit (rnso(df))
1 loop, best of 3: 6.27 s per loop

In [190]: %timeit (jez1(df))
100 loops, best of 3: 7.56 ms per loop

In [191]: %timeit (jez2(df))
100 loops, best of 3: 8.77 ms per loop

EDIT by another answer:

There is problem you need call duplicated by column Fruit and index. So there are 2 possible solutions:

create column from index by reset_index and call DataFrame.duplicated, last convert output to numpy array by values
add column Fruit to index by set_index and call Index.duplicated

#solution1
mask = df.reset_index().duplicated(['index','Fruit']).values
#solution2
#mask = df.set_index('Fruit', append=True).index.duplicated()
df.loc[mask, 'Quantity'] = 0

Timings1

def jez1(df):
    m = df.rename_axis('Date').groupby(['Date', 'Fruit']).cumcount().eq(0)
    df['Quantity'] = df['Quantity'].where(m, 0)
    return df

def jez3(df):
    mask = df.reset_index().duplicated(['index','Fruit']).values
    df.loc[mask, 'Quantity'] = 0
    return df

def jez4(df):
    mask = df.set_index('Fruit', append=True).index.duplicated()
    df.loc[mask, 'Quantity'] = 0
    return df

print (jez1(df))
print (jez3(df))
print (jez4(df))

In [268]: %timeit jez1(df)
100 loops, best of 3: 6.37 ms per loop

In [269]: %timeit jez3(df)
100 loops, best of 3: 3.82 ms per loop

In [270]: %timeit jez4(df)
100 loops, best of 3: 4.21 ms per loop

回答2:

One can merge Fruit and date in index and use for loop to convert rest quantity values to 0:

df['date_fruit'] = df.index+df.Fruit # new column with date and fruit merged
dflist = pd.unique(df.date_fruit)    # find its unique values
dfv = df.values                      # get rows as list of lists
for i in dflist:                     # for each unique date-fruit combination
    done = False
    for c in range(len(dfv)): 
        if dfv[c][2] == i:           # check each row
            if done: 
                dfv[c][1] = 0        # if not first, make quantity as 0
            else: 
                done = True

# create new dataframe with new data: 
newdf = pd.DataFrame(data=dfv, columns=df.columns, index=df.index)
newdf = newdf.iloc[:,:2]             # remove merged date-fruit column
print(newdf)

Output:

          Fruit Quantity
01/02/10  Apple        4
01/02/10  Apple        0
01/02/10   Pear        7
01/02/10  Grape        8
01/02/10  Grape        0
02/02/10  Apple        2
02/02/10  Fruit        6
02/02/10   Pear        8
02/02/10   Pear        0

回答3:

I read the OP's problem and I do not see the need of using pd.groupby() :

sort the data by days, fruit and quantity
call pd.Series.duplicated() to get the localization of duplicated fruit for each day
replace those values by 0

Since we only use duplicated it is way much faster.

Timings

import pandas as pd
import numpy as np

np.random.seed(1235)

N = 1000000
L = ['Apple','Pear','Grape','Fruit']
index = np.repeat(pd.date_range('2017-010-01', periods=N/20).strftime('%d/%m/%y'), 20)
df = (pd.DataFrame({'Fruit': np.random.choice(L, N),
                   'Quantity':np.random.randint(100, size=N), 'index':idx})
      .sort_values(['index', 'Fruit', 'Quantity'], ascending=[True, True, False])
      .set_index(['index', 'Fruit']))            

%%timeit
duplicated_mask = df.index.duplicated()
df.loc[duplicated_mask] = 0
[out] 13.4 ms ± 66.3 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)

%%timeit
def jez1(df):
    m = df.rename_axis('Date').groupby(['Date', 'Fruit']).cumcount().eq(0)
    df['Quantity'] = df['Quantity'].where(m, 0)
    return df
#df.reset_index('Fruit', inplace=True)
jez1(df)
[out] 136 ms ± 3.7 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)

来源：https://stackoverflow.com/questions/48126766/pandas-groupby-selecting-only-one-value-based-on-2-groups-and-converting-rest-to

标签

python

pandas

pandas-groupby

Pandas groupby selecting only one value based on 2 groups and converting rest to 0

问题

回答1:

回答2:

回答3:

Timings