Directory transfers with Paramiko

Question

How do you use Paramiko to transfer complete directories? I\'m trying to use:

sftp.put(\"/Folder1\",\"/Folder2\")

which is giving me this e

Answer 1

Paramiko does not support directory transfers on its own. You have to implement it, as many existing answers here show.

Or you can use pysftp. It's a wrapper around Paramiko that has more Python-ish look and feel and supports recursive operations. See

• pysftp.Connection.put_r()
• pysftp.Connection.get_r()

Or you can just base your code on pysftp source code. Full a standalone portable Paramiko-only code see my answers to:

• Python pysftp get_r from Linux works fine on Linux but not on Windows
• Python pysftp put_r does not work on Windows

And as my answers above show, you actually have to use your own code, if you are on Windows, as pysftp does not work there.

Answer 2

This is my first StackOverflow answer. I had a task today which is similar to this. So, I tried to find a direct way to copy entire folder from windows to linux using python and paramiko. After a little research, I came up with this solution which works for smaller size folders with subfolders and files in it.

This solution first makes the zip file for the current folder (os.walk() is very much helpful here), then copies to destination server and unzip there.

zipHere = zipfile.ZipFile("file_to_copy.zip", "w")

for root, folders, files in os.walk(FILE_TO_COPY_PATH):
    for file in files:
        zipHere.write(os.path.join(root, file), arcname=os.path.join(os.path.relpath(root, os.path.dirname(FILE_TO_COPY_PATH)), file))
    for folder in folders:
        zipHere.write(os.path.join(root, folder), arcname=os.path.join(os.path.relpath(root, os.path.dirname(FILE_TO_COPY_PATH)), folder))
zipHere.close()

# sftp is the paramiko.SFTPClient connection
sftp.put('local_zip_file_location','remote_zip_file_location')

# telnet_conn is the telnetlib.Telnet connection
telnet_conn.write('cd cd_to_zip_file_location')
telnet_conn.write('unzip -o file_to_copy.zip')

Answer 3

You'll need to do this just like you would locally with python (if you weren't using shutils).

Combine os.walk(), with sftp.mkdir() and sftp.put(). You may also want to check each file and directory with os.path.islink() depending on whether you want to resolve symlinks or not.

Answer 4

You can subclass paramiko.SFTPClient and add the following method to it:

import paramiko
import os

class MySFTPClient(paramiko.SFTPClient):
    def put_dir(self, source, target):
        ''' Uploads the contents of the source directory to the target path. The
            target directory needs to exists. All subdirectories in source are 
            created under target.
        '''
        for item in os.listdir(source):
            if os.path.isfile(os.path.join(source, item)):
                self.put(os.path.join(source, item), '%s/%s' % (target, item))
            else:
                self.mkdir('%s/%s' % (target, item), ignore_existing=True)
                self.put_dir(os.path.join(source, item), '%s/%s' % (target, item))

    def mkdir(self, path, mode=511, ignore_existing=False):
        ''' Augments mkdir by adding an option to not fail if the folder exists  '''
        try:
            super(MySFTPClient, self).mkdir(path, mode)
        except IOError:
            if ignore_existing:
                pass
            else:
                raise

To use it:

transport = paramiko.Transport((HOST, PORT))
transport.connect(username=USERNAME, password=PASSWORD)
sftp = MySFTPClient.from_transport(transport)
sftp.mkdir(target_path, ignore_existing=True)
sftp.put_dir(source_path, target_path)
sftp.close()

Answer 5

This can all be done quite easily using just paramiko.

A high level summary of the code below is:
- connect to the SFTP (steps 1 to 3)
- specify your source and target folders. (step 4)
- copy them over one by one to wherever you like (I've sent them to /tmp/). (step 5)

import paramiko

# 1 - Open a transport
host="your-host-name"
port = port_number
transport = paramiko.Transport((host, port))

# 2 - Auth
password="sftp_password"
username="sftp_username"
transport.connect(username = username, password = password)

# 3 - Go!

sftp = paramiko.SFTPClient.from_transport(transport)

# 4 - Specify your source and target folders.
source_folder="some/folder/path/on/sftp"
inbound_files=sftp.listdir(source_folder)

# 5 - Download all files from that path
for file in inbound_files :
    filepath = source_folde+file
    localpath = "/tmp/"+file
    sftp.get(filepath, localpath)

Answer 6

You might replace sftp = self.client.open_sftp() with paramiko's one and get rid of libcloud here.

import os.path
from stat import S_ISDIR
from libcloud.compute.ssh import SSHClient
from paramiko.sftp import SFTPError

class CloudSSHClient(SSHClient):


    @staticmethod
    def normalize_dirpath(dirpath):
        while dirpath.endswith("/"):
            dirpath = dirpath[:-1]
        return dirpath


    def mkdir(self, sftp, remotepath, mode=0777, intermediate=False):
        remotepath = self.normalize_dirpath(remotepath)
        if intermediate:
            try:
                sftp.mkdir(remotepath, mode=mode)
            except IOError, e:
                self.mkdir(sftp, remotepath.rsplit("/", 1)[0], mode=mode,
                           intermediate=True)
                return sftp.mkdir(remotepath, mode=mode)
        else:
            sftp.mkdir(remotepath, mode=mode)


    def put_dir_recursively(self,  localpath, remotepath, preserve_perm=True):
        "upload local directory to remote recursively"

        assert remotepath.startswith("/"), "%s must be absolute path" % remotepath

        # normalize
        localpath = self.normalize_dirpath(localpath)
        remotepath = self.normalize_dirpath(remotepath)

        sftp = self.client.open_sftp()

        try:
            sftp.chdir(remotepath)
            localsuffix = localpath.rsplit("/", 1)[1]
            remotesuffix = remotepath.rsplit("/", 1)[1]
            if localsuffix != remotesuffix:
                remotepath = os.path.join(remotepath, localsuffix)
        except IOError, e:
            pass

        for root, dirs, fls in os.walk(localpath):
            prefix = os.path.commonprefix([localpath, root])
            suffix = root.split(prefix, 1)[1]
            if suffix.startswith("/"):
                suffix = suffix[1:]

            remroot = os.path.join(remotepath, suffix)

            try:
                sftp.chdir(remroot)
            except IOError, e:
                if preserve_perm:
                    mode = os.stat(root).st_mode & 0777
                else:
                    mode = 0777
                self.mkdir(sftp, remroot, mode=mode, intermediate=True)
                sftp.chdir(remroot)

            for f in fls:
                remfile = os.path.join(remroot, f)
                localfile = os.path.join(root, f)
                sftp.put(localfile, remfile)
                if preserve_perm:
                    sftp.chmod(remfile, os.stat(localfile).st_mode & 0777)