Add notebook to perform full directory transfers using FTS

%% Cell type:code id:297d3556-9bef-4337-9880-aff634a5d48c tags:

``` python

import os

import sys

import stat

from datetime import datetime

import gfal2

```

%% Cell type:markdown id:3e903939-85d6-494e-ae15-c4f47fcdf2c5 tags:

# Setup

%% Cell type:markdown id:ab72b5e6-e7f4-467a-ada9-f8c983c86f88 tags:

## Environment

Need to have an environment with these packages in order to have the CLI scripts and the python bindings to manage the transfers.

* python=3.12

* gfal2-util

* python-gfal2

* fts3

* ipykernel

%% Cell type:markdown id:6fc89d0a-8a96-43c0-87d5-0b07a22b1427 tags:

## OIDC token

Steps to generate the token to be used for authentication

```

eval `oidc-agent`

oidc-gen -m --flow=device --no-webserver --iss=https://indigoiam-test.pic.es/ --client-secret=XXX --client-id=6679edd3-c74a-4e75-8774-0fc0cc72799d  --scope="profile openid offline_access storage.modify:/ storage.create:/ storage.read:/" fts_client_incaem001

export BEARER_TOKEN=$(oidc-token fts_client_incaem001)

```

Copy paste the BEARER_TOKEN in the following cell.

%% Cell type:code id:275ec614-a285-435c-a8e4-d18bf6612ca5 tags:

``` python

TOKEN = "eyJraWQiOiJyc2ExIiwiYWxnIjoiUlMyNTYifQ.eyJ3bGNnLnZlciI6IjEuMCIsInN1YiI6IjNjNDljOTEyLWEyNWYtYTc5NS02NTBmLTg1ODNkOWU4YTBlOCIsImF1ZCI6Imh0dHBzOlwvXC93bGNnLmNlcm4uY2hcL2p3dFwvdjFcL2FueSIsIm5iZiI6MTcyNTM2NTg1MSwic2NvcGUiOiJzdG9yYWdlLmNyZWF0ZTpcLyBvcGVuaWQgb2ZmbGluZV9hY2Nlc3MgcHJvZmlsZSBzdG9yYWdlLnJlYWQ6XC8gc3RvcmFnZS5tb2RpZnk6XC8iLCJpc3MiOiJodHRwczpcL1wvaW5kaWdvaWFtLXRlc3QucGljLmVzXC8iLCJleHAiOjE3MjUzNjk0NTEsImlhdCI6MTcyNTM2NTg1MSwianRpIjoiMzI3OGI5ODAtYTM0MC00OTJmLTk4MTUtYjU4M2FjM2ExMWE3IiwiY2xpZW50X2lkIjoiNjY3OWVkZDMtYzc0YS00ZTc1LTg3NzQtMGZjMGNjNzI3OTlkIn0.WgH2rff3vBaE45GADjCOh4TxZL0NyVDID7U4VYKej5A1ZnmFCbcVY6EUMcMsCDAE-NpgEPWw_F7cKwYf6CX6IzGk73SZ6VAQxTimDlF4BqVe7QIMtDkBkW4ZVyVsb_Yq1HlD8C5ORDTA2nU4a4Z0weJVCd_rotS_nWmwBAwhcjc"

```

%% Cell type:code id:c9a9084c-e243-4ff3-bc4e-ea53f71f984b tags:

``` python

# Origin endpoint / folder

ORIGIN_RSE = "https://globus-xrootd.cells.es:1094/20201125_multislice_largeBandwidth_zRange10um"

# Destination endpoint / folder

DES_RSE = "https://door04.pic.es:8460/pnfs/pic.es/data/incaem/20201125_multislice_largeBandwidth_zRange10um"

# FTS server to submit the transfer to

FTS_SERVER = 'https://fts02.pic.es:8446'

```

%% Cell type:code id:e0f2d3c4-419e-4963-9fac-82d2cafc3244 tags:

``` python

os.environ['BEARER_TOKEN'] = TOKEN

```

%% Cell type:markdown id:d39e771e-5612-4bf5-a19d-e819688c8004 tags:

# Build the list of files to be transferred

%% Cell type:code id:424340b3-2e9a-42f7-9219-d3c40707cdd7 tags:

``` python

ctx = gfal2.creat_context()

```

%% Cell type:markdown id:04fbd022-47ad-4362-9326-e372caf5033c tags:

Code in the following cell was picked from [here](https://github.com/cern-fts/gfal2-python/blob/develop/example/python/gfal2_recursive_ls.py) and modified to enable returning a list containing all origin file urls.

%% Cell type:code id:dd6960de-cd22-44a4-9c1b-7973d8d85353 tags:

``` python

class Crawler:

    def __init__(self, output='long', recursive=False,

                 max_levels=2, context=gfal2.creat_context()):

        self.context = context

        self.output = output # one of long, short, list

        self.recursive = recursive

        self.max_levels = max_levels

        # initialize empty file list filled by the crawl method

        self.file_list = []

    def _short_format(self, fname):

        return fname

    def _mode2str(self, mode):

        s = ['-'] * 10

        if stat.S_ISDIR(mode):

            s[0] = 'd'

        elif stat.S_ISLNK(mode):

            s[0] = 'l'

        elif not stat.S_ISREG(mode):

            s[0] = '?'

        for i in range(3):

            if mode & stat.S_IRUSR:

                s[1 + i * 3] = 'r'

            if mode & stat.S_IWUSR:

                s[2 + i * 3] = 'w'

            if mode & stat.S_IXUSR:

                s[3 + i * 3] = 'x'

            mode = mode << 3

        return ''.join(s)

    def _long_format(self, fname, fstat):

        return "%s %3d %5d %5d %10d %s %s" % \

            (self._mode2str(fstat.st_mode),

             fstat.st_nlink, fstat.st_uid, fstat.st_gid,

             fstat.st_size,

             datetime.fromtimestamp(fstat.st_mtime).strftime('%b %d %H:%M'),

             fname)

    def _crawl(self, url, out, level=0):

        if level > self.max_levels:

            return

        tabbing = '\n' + '  ' * level

        try:

            entries = [f for f in self.context.listdir(url)

                       if f != '.' and f != '..']

        except gfal2.GError:

            out.write(tabbing + '!')

            return

        for f in entries:

            full = url + '/' + f

            # Do the stat only if we need to

            if self.recursive or self.long:

                try:

                    fstat = self.context.stat(full)

                except gfal2.GError:

                    fstat = self.context.st_stat()

            # Print entry

            if self.output == 'long':

                out.write(tabbing + self._long_format(f, fstat))

            elif self.output == 'short':

                out.write(tabbing + self._short_format(f))

            elif self.output == 'list' and not stat.S_ISDIR(fstat.st_mode):

                self.file_list.append(full)

            # Descend

            if self.recursive and stat.S_ISDIR(fstat.st_mode):

                self._crawl(full, out, level + 1)

    def crawl(self, url, out=sys.stdout):

        self.file_list = []

        self._crawl(url, out)

        return self.file_list

```

%% Cell type:code id:253faade-2678-49f3-863b-1163956023be tags:

``` python

crawler = Crawler(output="list", recursive=True, context=ctx, max_levels=float("inf"))

```

%% Cell type:code id:eae48c45-4c1e-4f77-aa9e-839d8465110d tags:

``` python

origin_file_list = crawler.crawl(ORIGIN_RSE)

```

%% Cell type:code id:96927935-165c-4595-b508-17040e4aec30 tags:

``` python

len(origin_file_list)

```

%% Output

    737

%% Cell type:code id:ee98d603-3080-447a-8568-7b85fda10bf4 tags:

``` python

origin_file_list[0]

```

%% Output

    'https://globus-xrootd.cells.es:1094/20201125_multislice_largeBandwidth_zRange10um/imgs2ppt/dofmap_sirius_bandwidth116.67.png'

%% Cell type:markdown id:625c24b8-681a-44ce-8c75-1d2c4e265d89 tags:

# Submit the file transfers to FTS

Following the instructions in the FTS3 [easy bindings documentation](https://github.com/cern-fts/fts-rest/tree/develop/docs/easy)

%% Cell type:code id:9dc83208-bba0-43e8-ac7e-b3c57f3194ce tags:

``` python

import fts3.rest.client.easy as fts3

```

%% Cell type:markdown id:950c849b-4389-46ca-bf89-9cb8ce4ca2c7 tags:

FTS only allows file transfers (not folders), so we have to build a list of transfers, file to file.

%% Cell type:code id:8676c4e0-c394-46e9-997d-06ac3a262002 tags:

``` python

transfer_list = [ fts3.new_transfer(

    source=f,

    destination=f.replace(ORIGIN_RSE, DES_RSE),

    checksum=None

) for f in origin_file_list ]

```

%% Cell type:code id:2400db17-d7f2-47a9-b570-0e91d972a6b3 tags:

``` python

fts_job = fts3.new_job(transfer_list)

```

%% Cell type:code id:5873deff-834a-45d4-af6a-858afba21cea tags:

``` python

fts_context = fts3.Context(

    FTS_SERVER,

    fts_access_token=os.environ['BEARER_TOKEN'],

    verify=False

)

```

%% Cell type:markdown id:6788d226-f022-4287-b12f-bf5ac2960822 tags:

Check that authentication with FTS is working

%% Cell type:code id:930e8f15-e7e5-4ee1-b9be-842d312d5e41 tags:

``` python

fts3.whoami(fts_context)

```

%% Output

    {'user_dn': '3c49c912-a25f-a795-650f-8583d9e8a0e8',

     'dn': ['3c49c912-a25f-a795-650f-8583d9e8a0e8'],

     'base_id': '01874efb-4735-4595-bc9c-591aef8240c9',

     'voms_cred': ['offline_access',

      'openid',

      'profile',

      'storage.create:/',

      'storage.modify:/',

      'storage.read:/'],

     'vos': ['indigoiam-test.pic.es'],

     'vos_id': ['f4656e4b-8f00-537c-8744-78a0d32b2dd4'],

     'roles': [],

     'level': {'transfer': 'vo', 'datamanagement': 'all'},

     'delegation_id': 'adf087ff3b34b961',

     'method': 'oauth2',

     'is_root': False,

     'oauth2_scope': 'offline_access openid profile storage.create:/ storage.modify:/ storage.read:/',

     'wlcg_profile': True,

     'get_granted_level_for_overriden': {},

     'get_granted_level_for': {}}

%% Cell type:markdown id:ee767e97-b718-4d6e-a894-1c8846818957 tags:

Submit the transfer to FTS

%% Cell type:code id:4829c6fb-20e4-4752-92ea-0541a43e1e5b tags:

``` python

job_id = fts3.submit(fts_context, fts_job)

```

%% Cell type:code id:fed364ba-a706-4099-b102-f282ee1d1744 tags:

``` python

job_id

```

%% Output

    '106e8c30-69f1-11ef-9268-001a4aae003f'

%% Cell type:code id:73ffbeba-742c-4b32-9372-fcfa665ec989 tags:

``` python

fts3.get_job_status(fts_context, job_id)

```

%% Output

    {'job_id': '106e8c30-69f1-11ef-9268-001a4aae003f',

     'user_dn': '3c49c912-a25f-a795-650f-8583d9e8a0e8',

     'dst_file_report': False,

     'source_se': 'https://globus-xrootd.cells.es',

     'cred_id': 'adf087ff3b34b961',

     'overwrite_flag': False,

     'job_metadata': {'auth_method': 'oauth2'},

     'dest_se': 'https://door04.pic.es',

     'vo_name': 'indigoiam-test.pic.es',

     'job_finished': None,

     'retry': -1,

     'source_spacetoken': None,

     'job_state': 'ACTIVE',

     'reason': None,

     'copy_pin_lifetime': -1,

     'retry_delay': 0,

     'destination_spacetoken': None,

     'job_type': 'N',

     'submit_time': '2024-09-03T12:35:56',

     'bring_online': -1,

     'os_project_id': None,

     'verify_checksum': 'n',

     'cancel_job': False,

     'priority': 3,

     'archive_timeout': -1,

     'submit_host': 'fts02.pic.es',

     'max_time_in_queue': None,

     'target_qos': None,

     'internal_job_params': 'nostreams:1',

     'http_status': '200 Ok'}

%% Cell type:code id:8cf9cbdd-35b8-4a23-beb6-0e0a4874d774 tags:

``` python

```

# ---

# jupyter:

#   jupytext:

#     formats: ipynb,py:light

#     text_representation:

#       extension: .py

#       format_name: light

#       format_version: '1.5'

#       jupytext_version: 1.16.2

#   kernelspec:

#     display_name: transfers

#     language: python

#     name: transfers

# ---

# +

import os

import sys

import stat

from datetime import datetime

import gfal2

# -

# # Setup

# ## Environment

#

# Need to have an environment with these packages in order to have the CLI scripts and the python bindings to manage the transfers.

#

# * python=3.12

# * gfal2-util

# * python-gfal2

# * fts3

# * ipykernel

#

#

# ## OIDC token

#

# Steps to generate the token to be used for authentication

# ```

# eval `oidc-agent`

# oidc-gen -m --flow=device --no-webserver --iss=https://indigoiam-test.pic.es/ --client-secret=XXX --client-id=6679edd3-c74a-4e75-8774-0fc0cc72799d  --scope="profile openid offline_access storage.modify:/ storage.create:/ storage.read:/" fts_client_incaem001

# export BEARER_TOKEN=$(oidc-token fts_client_incaem001)

# ```

#

# Copy paste the BEARER_TOKEN in the following cell.

TOKEN = "eyJraWQiOiJyc2ExIiwiYWxnIjoiUlMyNTYifQ.eyJ3bGNnLnZlciI6IjEuMCIsInN1YiI6IjNjNDljOTEyLWEyNWYtYTc5NS02NTBmLTg1ODNkOWU4YTBlOCIsImF1ZCI6Imh0dHBzOlwvXC93bGNnLmNlcm4uY2hcL2p3dFwvdjFcL2FueSIsIm5iZiI6MTcyNTM2NTg1MSwic2NvcGUiOiJzdG9yYWdlLmNyZWF0ZTpcLyBvcGVuaWQgb2ZmbGluZV9hY2Nlc3MgcHJvZmlsZSBzdG9yYWdlLnJlYWQ6XC8gc3RvcmFnZS5tb2RpZnk6XC8iLCJpc3MiOiJodHRwczpcL1wvaW5kaWdvaWFtLXRlc3QucGljLmVzXC8iLCJleHAiOjE3MjUzNjk0NTEsImlhdCI6MTcyNTM2NTg1MSwianRpIjoiMzI3OGI5ODAtYTM0MC00OTJmLTk4MTUtYjU4M2FjM2ExMWE3IiwiY2xpZW50X2lkIjoiNjY3OWVkZDMtYzc0YS00ZTc1LTg3NzQtMGZjMGNjNzI3OTlkIn0.WgH2rff3vBaE45GADjCOh4TxZL0NyVDID7U4VYKej5A1ZnmFCbcVY6EUMcMsCDAE-NpgEPWw_F7cKwYf6CX6IzGk73SZ6VAQxTimDlF4BqVe7QIMtDkBkW4ZVyVsb_Yq1HlD8C5ORDTA2nU4a4Z0weJVCd_rotS_nWmwBAwhcjc"

# +

# Origin endpoint / folder

ORIGIN_RSE = "https://globus-xrootd.cells.es:1094/20201125_multislice_largeBandwidth_zRange10um"

# Destination endpoint / folder

DES_RSE = "https://door04.pic.es:8460/pnfs/pic.es/data/incaem/20201125_multislice_largeBandwidth_zRange10um"

# FTS server to submit the transfer to

FTS_SERVER = 'https://fts02.pic.es:8446'

# -

os.environ['BEARER_TOKEN'] = TOKEN

# # Build the list of files to be transferred

ctx = gfal2.creat_context()

# Code in the following cell was picked from [here](https://github.com/cern-fts/gfal2-python/blob/develop/example/python/gfal2_recursive_ls.py) and modified to enable returning a list containing all origin file urls.

class Crawler:

    def __init__(self, output='long', recursive=False,

                 max_levels=2, context=gfal2.creat_context()):

        self.context = context

        self.output = output # one of long, short, list

        self.recursive = recursive

        self.max_levels = max_levels

        # initialize empty file list filled by the crawl method

        self.file_list = []

    def _short_format(self, fname):

        return fname

    def _mode2str(self, mode):

        s = ['-'] * 10

        if stat.S_ISDIR(mode):

            s[0] = 'd'

        elif stat.S_ISLNK(mode):

            s[0] = 'l'

        elif not stat.S_ISREG(mode):

            s[0] = '?'

        for i in range(3):

            if mode & stat.S_IRUSR:

                s[1 + i * 3] = 'r'

            if mode & stat.S_IWUSR:

                s[2 + i * 3] = 'w'

            if mode & stat.S_IXUSR:

                s[3 + i * 3] = 'x'

            mode = mode << 3

        return ''.join(s)

    def _long_format(self, fname, fstat):

        return "%s %3d %5d %5d %10d %s %s" % \

            (self._mode2str(fstat.st_mode),

             fstat.st_nlink, fstat.st_uid, fstat.st_gid,

             fstat.st_size,

             datetime.fromtimestamp(fstat.st_mtime).strftime('%b %d %H:%M'),

             fname)

    def _crawl(self, url, out, level=0):

        if level > self.max_levels:

            return

        tabbing = '\n' + '  ' * level

        try:

            entries = [f for f in self.context.listdir(url)

                       if f != '.' and f != '..']

        except gfal2.GError:

            out.write(tabbing + '!')

            return

        for f in entries:

            full = url + '/' + f

            # Do the stat only if we need to

            if self.recursive or self.long:

                try:

                    fstat = self.context.stat(full)

                except gfal2.GError:

                    fstat = self.context.st_stat()

            # Print entry

            if self.output == 'long':

                out.write(tabbing + self._long_format(f, fstat))

            elif self.output == 'short':

                out.write(tabbing + self._short_format(f))

            elif self.output == 'list' and not stat.S_ISDIR(fstat.st_mode):

                self.file_list.append(full)

            # Descend

            if self.recursive and stat.S_ISDIR(fstat.st_mode):

                self._crawl(full, out, level + 1)

    def crawl(self, url, out=sys.stdout):

        self.file_list = []

        self._crawl(url, out)

        return self.file_list

crawler = Crawler(output="list", recursive=True, context=ctx, max_levels=float("inf"))

origin_file_list = crawler.crawl(ORIGIN_RSE)

len(origin_file_list)

origin_file_list[0]

# # Submit the file transfers to FTS

#

# Following the instructions in the FTS3 [easy bindings documentation](https://github.com/cern-fts/fts-rest/tree/develop/docs/easy)

import fts3.rest.client.easy as fts3

# FTS only allows file transfers (not folders), so we have to build a list of transfers, file to file.

transfer_list = [ fts3.new_transfer(

    source=f,

    destination=f.replace(ORIGIN_RSE, DES_RSE),

    checksum=None

) for f in origin_file_list ]

fts_job = fts3.new_job(transfer_list)

fts_context = fts3.Context(

    FTS_SERVER,

    fts_access_token=os.environ['BEARER_TOKEN'],

    verify=False

)

# Check that authentication with FTS is working

fts3.whoami(fts_context)

# Submit the transfer to FTS

job_id = fts3.submit(fts_context, fts_job)

job_id

fts3.get_job_status(fts_context, job_id)