Merge sftp_processor.py and ftp_processor.py

[danleyb2]

Most of the params look similar

[marcbelmont]

@adolfoarmas, could you please take a look at this? Is it something you can do?

[adolfoarmas]

@adolfoarmas, could you please take a look at this? Is it something you can do?

@marcbelmont after taking a look, I think I can do it

[marcbelmont]

@adolfoarmas great. Ping me when the PR is ready.

[marcbelmont]

Here's the gpt4 solution. No idea if it works...

#!/usr/bin/env python
import argparse
import json
import logging
import os
import sys
import tempfile
import time
from datetime import datetime, timedelta
from ftplib import FTP
from pathlib import Path
from stat import S_ISDIR, S_ISREG

import paramiko
from plate_recognition import recognition_api, save_results

logging.basicConfig(format="%(message)s", level=logging.INFO)
LOG_LEVEL = os.environ.get("LOGGING", "INFO").upper()

# Keep track of processed file names
processed = []

def get_connection(hostname, username, port, password=None, pkey_path=None):
    ssh = paramiko.SSHClient()
    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())

    if password:
        ssh.connect(hostname, port, username, password)
    else:
        key = paramiko.RSAKey.from_private_key_file(pkey_path)
        ssh.connect(hostname, port, username, pkey=key)

    sftp = ssh.open_sftp()
    return sftp

def track_processed(args):
    """
    Track processed if there might be a delete timeout or interval specified

    :param args:
    :return:
    """
    return args.delete or (args.interval and args.interval > 0)

def process_files(ftp_client, ftp_files, args):
    """
    Process a list of file paths by:
    1. Deletes old files in ftp_files from  ftp_client
    2. For new files, retrieving the full file from ftp_client
    3. Calling Snapshot API and Tracking Successfully Processed file paths

    :param ftp_client:
    :param ftp_files: List of files in the format [path, modified datetime], usually from a single folder.
    :param args:
    :return:
    """

    results = []

    for file_last_modified in ftp_files:
        ftp_file = file_last_modified[0]
        last_modified = file_last_modified[1]

        if track_processed(args) and ftp_file in processed:
            if args.delete is not None:
                rm_older_than_date = datetime.now() - timedelta(seconds=args.delete)
                if rm_older_than_date > last_modified:
                    ftp_client.delete(ftp_file)
                    processed.remove(ftp_file)

            continue

        logging.info(ftp_file)
        with tempfile.NamedTemporaryFile(suffix="_" + ftp_file, mode="rb+") as image:
            ftp_client.retrbinary("RETR " + ftp_file, image.write)
            api_res = recognition_api(
                image,
                args.regions,
                args.api_key,
                args.sdk_url,
                camera_id=args.camera_id,
                timestamp=args.timestamp,
                mmc=args.mmc,
                exit_on_error=False,
            )
            results.append(api_res)

        if track_processed(args):
            processed.append(ftp_file)

    if args.output_file:
        save_results(results, args)
    else:
        print(json.dumps(results, indent=2))

def main(args):
    """

    :param args: args from argparse
    :return:
    """
    sftp = None
    try:
        sftp = get_connection(
            args.host, args.user, args.port, password=args.password, pkey_path=args.pkey
        )
        root = Path(args.folder)

        for entry in sftp.listdir_attr(str(root)):
            if track_processed(args) and entry.filename in processed:
                lgr.debug(f"skip processed file: {entry.filename}")
                continue

            remote_path = root / entry.filename
            mode = entry.st_mode
            if S_ISDIR(mode):
                # Skip Dir
                pass
            elif S_ISREG(mode):
                with tempfile.NamedTemporaryFile(
                    suffix="_" + entry.filename, mode="rb+"
                ) as image:
                    sftp.getfo(str(remote_path), image)

                    api_res = recognition_api(
                        image,
                        args.regions,
                        args.api_token,
                        args.sdk_url,
                        camera_id=args.camera_id,
                        mmc=args.mmc,
                        exit_on_error=False,
                    )

                    lgr.info(api_res)
                    if args.delete and api_res:
                        sftp.remove(str(remote_path))

                if track_processed(args):
                    processed.append(entry.filename)

    except Exception as e:
        lgr.error(e)
    finally:
        if sftp:
            sftp.close()

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="Read license plates from the images on an SFTP server and output the result as JSON or CSV.",
        epilog="",
        formatter_class=argparse.RawTextHelpFormatter,
    )
    # Add all the arguments here
    cli_args = parser.parse_args()

    if not cli_args.sdk_url and not cli_args.api_token:
        raise Exception("api-key is required")

    if cli_args.interval and cli_args.interval > 0:
        while True:
            main(cli_args)
            time.sleep(cli_args.interval)
    else:
        main(cli_args)

[adolfoarmas]

Great, thanks! Allow me to see how i can merge it with the progress i have so far.

[marcbelmont]

@adolfoarmas, can you open a PR with your changes?