Directory transfers with Paramiko

Question

How do you use Paramiko to transfer complete directories? I'm trying to use:

sftp.put("/Folder1","/Folder2")

which is giving me this error -

Error : [Errno 21] Is a directory

Solution 1

You'll need to do this just like you would locally with python (if you weren't using shutils).

Combine os.walk(), with sftp.mkdir() and sftp.put(). You may also want to check each file and directory with os.path.islink() depending on whether you want to resolve symlinks or not.

Solution 2

You can subclass paramiko.SFTPClient and add the following method to it:

import paramiko
import os

class MySFTPClient(paramiko.SFTPClient):
    def put_dir(self, source, target):
        ''' Uploads the contents of the source directory to the target path. The
            target directory needs to exists. All subdirectories in source are 
            created under target.
        '''
        for item in os.listdir(source):
            if os.path.isfile(os.path.join(source, item)):
                self.put(os.path.join(source, item), '%s/%s' % (target, item))
            else:
                self.mkdir('%s/%s' % (target, item), ignore_existing=True)
                self.put_dir(os.path.join(source, item), '%s/%s' % (target, item))

    def mkdir(self, path, mode=511, ignore_existing=False):
        ''' Augments mkdir by adding an option to not fail if the folder exists  '''
        try:
            super(MySFTPClient, self).mkdir(path, mode)
        except IOError:
            if ignore_existing:
                pass
            else:
                raise

To use it:

transport = paramiko.Transport((HOST, PORT))
transport.connect(username=USERNAME, password=PASSWORD)
sftp = MySFTPClient.from_transport(transport)
sftp.mkdir(target_path, ignore_existing=True)
sftp.put_dir(source_path, target_path)
sftp.close()

Solution 3

Here's my piece of code:

import errno
import os
import stat

def download_files(sftp_client, remote_dir, local_dir):
    if not exists_remote(sftp_client, remote_dir):
        return

    if not os.path.exists(local_dir):
        os.mkdir(local_dir)

    for filename in sftp_client.listdir(remote_dir):
        if stat.S_ISDIR(sftp_client.stat(remote_dir + filename).st_mode):
            # uses '/' path delimiter for remote server
            download_files(sftp_client, remote_dir + filename + '/', os.path.join(local_dir, filename))
        else:
            if not os.path.isfile(os.path.join(local_dir, filename)):
                sftp_client.get(remote_dir + filename, os.path.join(local_dir, filename))


def exists_remote(sftp_client, path):
    try:
        sftp_client.stat(path)
    except IOError, e:
        if e.errno == errno.ENOENT:
            return False
        raise
    else:
        return True

Solution 4

This can all be done quite easily using just paramiko.

A high level summary of the code below is:
- connect to the SFTP (steps 1 to 3)
- specify your source and target folders. (step 4)
- copy them over one by one to wherever you like (I've sent them to /tmp/). (step 5)

import paramiko

# 1 - Open a transport
host="your-host-name"
port = port_number
transport = paramiko.Transport((host, port))

# 2 - Auth
password="sftp_password"
username="sftp_username"
transport.connect(username = username, password = password)

# 3 - Go!

sftp = paramiko.SFTPClient.from_transport(transport)

# 4 - Specify your source and target folders.
source_folder="some/folder/path/on/sftp"
inbound_files=sftp.listdir(source_folder)

# 5 - Download all files from that path
for file in inbound_files :
    filepath = source_folder+file
    localpath = "/tmp/"+file
    sftp.get(filepath, localpath)

Solution 5

You might replace sftp = self.client.open_sftp() with paramiko's one and get rid of libcloud here.

import os.path
from stat import S_ISDIR
from libcloud.compute.ssh import SSHClient
from paramiko.sftp import SFTPError

class CloudSSHClient(SSHClient):


    @staticmethod
    def normalize_dirpath(dirpath):
        while dirpath.endswith("/"):
            dirpath = dirpath[:-1]
        return dirpath


    def mkdir(self, sftp, remotepath, mode=0777, intermediate=False):
        remotepath = self.normalize_dirpath(remotepath)
        if intermediate:
            try:
                sftp.mkdir(remotepath, mode=mode)
            except IOError, e:
                self.mkdir(sftp, remotepath.rsplit("/", 1)[0], mode=mode,
                           intermediate=True)
                return sftp.mkdir(remotepath, mode=mode)
        else:
            sftp.mkdir(remotepath, mode=mode)


    def put_dir_recursively(self,  localpath, remotepath, preserve_perm=True):
        "upload local directory to remote recursively"

        assert remotepath.startswith("/"), "%s must be absolute path" % remotepath

        # normalize
        localpath = self.normalize_dirpath(localpath)
        remotepath = self.normalize_dirpath(remotepath)

        sftp = self.client.open_sftp()

        try:
            sftp.chdir(remotepath)
            localsuffix = localpath.rsplit("/", 1)[1]
            remotesuffix = remotepath.rsplit("/", 1)[1]
            if localsuffix != remotesuffix:
                remotepath = os.path.join(remotepath, localsuffix)
        except IOError, e:
            pass

        for root, dirs, fls in os.walk(localpath):
            prefix = os.path.commonprefix([localpath, root])
            suffix = root.split(prefix, 1)[1]
            if suffix.startswith("/"):
                suffix = suffix[1:]

            remroot = os.path.join(remotepath, suffix)

            try:
                sftp.chdir(remroot)
            except IOError, e:
                if preserve_perm:
                    mode = os.stat(root).st_mode & 0777
                else:
                    mode = 0777
                self.mkdir(sftp, remroot, mode=mode, intermediate=True)
                sftp.chdir(remroot)

            for f in fls:
                remfile = os.path.join(remroot, f)
                localfile = os.path.join(root, f)
                sftp.put(localfile, remfile)
                if preserve_perm:
                    sftp.chmod(remfile, os.stat(localfile).st_mode & 0777)

View more solutions