Capture all csv files within all subfolders in main directory - Python 3.x

问题

Below code is used to split csv files based on a given time value. The problem is this code won't capture all the csv files. For example inside TT1 folder there are several subfolders.And those subfolders have folders inside them. And within those sub-sub-folders there are csv files. When I give the path as path='/root/Desktop/TT1 it wont process all the files within those sub-sub-folders. How can I fix this please.

AFTER @Serafeim 's answer (https://stackoverflow.com/a/57110519/5025009), I tried this:

import pandas as pd
import numpy as np
import glob
import os

path = '/root/Desktop/TT1/'
mystep = 0.4

#define the function
def data_splitter(df, name):
    max_time = df['Time'].max() # get max value of Time for the current csv file (df)
    myrange= np.arange(0, max_time, mystep) # build the threshold range
    for k in range(len(myrange)):
        # build the upper values 
        temp = df[(df['Time'] >= myrange[k]) & (df['Time'] < myrange[k] + mystep)]
        temp.to_csv("/root/Desktop/T1/{}_{}.csv".format(name, k))

for filename in glob.glob(os.path.join(path, '*.csv')):
    df = pd.read_csv(filename)
    name = os.path.split(filename)[1] # get the name of the file
    data_splitter(df, name)

回答1:

You can get automatically all the subfolders and change the path: If all the subfolders start with "Sub":

import pandas as pd
import numpy as np
import glob
import os

path = '/root/Desktop/TT1/'
mystep = 0.4

#define the function
def data_splitter(df, name):
    max_time = df['Time'].max() # get max value of Time for the current csv file (df)
    myrange= np.arange(0, max_time, mystep) # build the threshold range
    for k in range(len(myrange)):
        # build the upper values 
        temp = df[(df['Time'] >= myrange[k]) & (df['Time'] < myrange[k] + mystep)]
        temp.to_csv("/root/Desktop/T1/{}_{}.csv".format(name, k))

# use os.walk(path) on the main path to get ALL subfolders inside path
for root,dirs,_ in os.walk(path):
    for d in dirs:
        path_sub = os.path.join(root,d) # this is the current subfolder
        for filename in glob.glob(os.path.join(path_sub, '*.csv')):
            df = pd.read_csv(filename)
            name = os.path.split(filename)[1] # get the name of the current csv file
            data_splitter(df, name)

来源：https://stackoverflow.com/questions/57180566/capture-all-csv-files-within-all-subfolders-in-main-directory-python-3-x

标签

python

python-3.x

pandas

csv

glob