magento oauth authentication is not working with apache airflow

sivajik34 commented 1 month ago

Summary

I'm trying to use apache airflow for data sync with oauth authentication. but it is not working. Getting a signature is an invalid type of error. https://github.com/sivajik34/magento-airflow/blob/main/src/airflow_magento_provider/airflow_magento_provider/hooks/magento_hooks.py

Examples

import urllib.parse

import hmac

import hashlib

import base64

import time

import uuid

from urllib.parse import urlencode

import requests

from airflow.hooks.base_hook import BaseHook

from airflow.exceptions import AirflowException

class MagentoHook(BaseHook):

def __init__(self, magento_conn_id='magento', *args, **kwargs):

    super().__init__(*args, **kwargs)

    self.magento_conn_id = magento_conn_id

    self.connection = self.get_connection(self.magento_conn_id)

    self._configure_oauth()

def _configure_oauth(self):

    """Configure OAuth authentication for Magento API"""

    conn = self.get_connection(self.magento_conn_id)

    self.consumer_key = conn.extra_dejson.get('consumer_key')

    self.consumer_secret = conn.extra_dejson.get('consumer_secret')

    self.access_token = conn.extra_dejson.get('access_token')

    self.access_token_secret = conn.extra_dejson.get('access_token_secret')

    if not all([self.consumer_key, self.consumer_secret, self.access_token, self.access_token_secret]):

        raise AirflowException("Magento OAuth credentials are not set properly in Airflow connection")

def _generate_oauth_parameters(self, url, method, data=None):

    """Generate OAuth parameters for the request"""

    oauth_nonce = str(uuid.uuid4())

    oauth_timestamp = str(int(time.time()))

    oauth_signature_method = 'HMAC-SHA256'

    oauth_version = '1.0'

    # Parse the URL to separate the query parameters

    parsed_url = urllib.parse.urlparse(url)

    query_params = urllib.parse.parse_qsl(parsed_url.query)

    # Prepare OAuth parameters

    oauth_params = {

        'oauth_consumer_key': self.consumer_key,

        'oauth_nonce': oauth_nonce,

        'oauth_signature_method': oauth_signature_method,

        'oauth_timestamp': oauth_timestamp,

        'oauth_token': self.access_token,

        'oauth_version': oauth_version

    }

    # Include data parameters if present

    if data:

        query_params.extend(data.items())

    # Combine OAuth and query/data parameters, and sort them

    all_params = oauth_params.copy()

    all_params.update(query_params)

    sorted_params = sorted(all_params.items(), key=lambda x: x[0])

    # Encode and create the parameter string

    param_str = urllib.parse.urlencode(sorted_params, safe='')

    # Construct the base string

    base_string = f"{method.upper()}&{urllib.parse.quote(parsed_url.scheme + '://' + parsed_url.netloc + parsed_url.path, safe='')}&{urllib.parse.quote(param_str, safe='')}"

    # Create the signing key

    signing_key = f"{urllib.parse.quote(self.consumer_secret, safe='')}&{urllib.parse.quote(self.access_token_secret, safe='')}"

    # Generate the OAuth signature

    oauth_signature = base64.b64encode(hmac.new(signing_key.encode(), base_string.encode(), hashlib.sha256).digest()).decode()

    # Add the signature to OAuth parameters

    oauth_params['oauth_signature'] = oauth_signature

    # Construct the Authorization header

    auth_header = 'OAuth ' + ', '.join([f'{urllib.parse.quote(k)}="{urllib.parse.quote(v)}"' for k, v in oauth_params.items()])        

    return auth_header

def get_request(self, endpoint, method='GET', data=None):

    """Perform an API request to Magento"""

    url = f"https://{self.connection.host}/rest/default/V1/{endpoint}"

    headers = {

        'Content-Type': 'application/json',

        'Authorization': self._generate_oauth_parameters(url, method, data)

    }

    try:

        if method.upper() == 'GET':

            response = requests.get(url, headers=headers, verify=False)  # Use GET method

        else:

            response = requests.request(method, url, headers=headers, json=data, verify=False)  # Use method provided

        response.raise_for_status()  # Will raise an error for bad responses

    except requests.exceptions.HTTPError as http_err:

        error_details = None

        try:

            # Attempt to parse and log the error response

            error_details = response.json() if response.content else {}

            self.log.error(f"Error details: {error_details}")

        except ValueError:

            # Failed to parse JSON response

            pass

        raise AirflowException(

            f"Request failed: {http_err}. Error details: {error_details}"

        )

    except requests.exceptions.RequestException as e:

        raise AirflowException(f"Request failed: {str(e)}")

    return response.json()

def get_orders(self, search_criteria=None):

    """

    Fetch orders from Magento.

    :param search_criteria: Dictionary containing search criteria to filter orders

    :return: List of orders

    """

    endpoint = "orders"

    if search_criteria:

        # Convert search criteria dictionary to URL parameters

        query_string = urlencode(search_criteria, doseq=True)

        endpoint = f"{endpoint}?{query_string}"

    self.log.info(f"Requesting orders from Magento: {endpoint}")

    return self.get_request(endpoint)

Proposed solution

No response

Release note

No response

Triage and priority

[ ] Severity: S0 - Affects critical data or functionality and leaves users without workaround.
[ ] Severity: S1 - Affects critical data or functionality and forces users to employ a workaround.
[X] Severity: S2 - Affects non-critical data or functionality and forces users to employ a workaround.
[X] Severity: S3 - Affects non-critical data or functionality and does not force users to employ a workaround.
[ ] Severity: S4 - Affects aesthetics, professional look and feel, “quality” or “usability”.

m2-assistant[bot] commented 1 month ago

Hi @sivajik34. Thank you for your report. To speed up processing of this issue, make sure that the issue is reproducible on the vanilla Magento instance following Steps to reproduce. To deploy vanilla Magento instance on our environment, Add a comment to the issue:

@magento give me 2.4-develop instance - upcoming 2.4.x release
For more details, review the Magento Contributor Assistant documentation.
Add a comment to assign the issue: @magento I am working on this
To learn more about issue processing workflow, refer to the Code Contributions.

Join Magento Community Engineering Slack and ask your questions in #github channel. :warning: According to the Magento Contribution requirements, all issues must go through the Community Contributions Triage process. Community Contributions Triage is a public meeting. :clock10: You can find the schedule on the Magento Community Calendar page. :telephone_receiver: The triage of issues happens in the queue order. If you want to speed up the delivery of your contribution, join the Community Contributions Triage session to discuss the appropriate ticket.

sivajik34 commented 1 month ago

now GET call is working fine, but POST call is not working.

from __future__ import annotations
import base64
import hashlib
import hmac
import time
import urllib.parse
import uuid
from urllib.parse import urlencode
import requests
from airflow.exceptions import AirflowException
from airflow.hooks.base import BaseHook

class MagentoHook(BaseHook):

    """Creates new connection to Magento and allows you to pull data from Magento and push data to Magento."""

    conn_name_attr = "magento_conn_id"

    default_conn_name = "magento_default"

    conn_type = "magento"

    hook_name = "Magento"

    BASE_URL = "/rest/default/V1"  # Common base URL for Magento API

    def __init__(self, magento_conn_id=default_conn_name):

        super().__init__()

        self.magento_conn_id = magento_conn_id

        self.connection = self.get_connection(self.magento_conn_id)

        self._configure_oauth()

        # Validate connection host

        if not self.connection.host:

            raise AirflowException("Magento connection host is not set properly in Airflow connection")

    def _configure_oauth(self):

        """Configure OAuth authentication for Magento API."""

        self.consumer_key = self.connection.extra_dejson.get("consumer_key")

        self.consumer_secret = self.connection.extra_dejson.get("consumer_secret")

        self.access_token = self.connection.extra_dejson.get("access_token")

        self.access_token_secret = self.connection.extra_dejson.get("access_token_secret")

        if not all([self.consumer_key, self.consumer_secret, self.access_token, self.access_token_secret]):

            raise AirflowException("Magento OAuth credentials are not set properly in Airflow connection")

    def _generate_oauth_parameters(self, url, method, data=None):

        """Generate OAuth parameters for the request."""

        oauth_nonce = str(uuid.uuid4())

        oauth_timestamp = str(int(time.time()))

        oauth_signature_method = "HMAC-SHA256"

        oauth_version = "1.0"

        # Parse the URL to separate the query parameters

        parsed_url = urllib.parse.urlparse(url)

        query_params = urllib.parse.parse_qsl(parsed_url.query)

        # Prepare OAuth parameters

        oauth_params = {

            "oauth_consumer_key": self.consumer_key,

            "oauth_nonce": oauth_nonce,

            "oauth_signature_method": oauth_signature_method,

            "oauth_timestamp": oauth_timestamp,

            "oauth_token": self.access_token,

            "oauth_version": oauth_version,

        }

        # Include data parameters if present

        if data:

            if method.upper() == "POST":

                # POST data should be included in the base string

                data_params = urlencode(data, doseq=True)

                query_params.extend(urllib.parse.parse_qsl(data_params))

        # Combine OAuth and query/data parameters, and sort them

        all_params = oauth_params.copy()

        all_params.update(query_params)

        sorted_params = sorted(all_params.items(), key=lambda x: x[0])

        # Encode and create the parameter string

        param_str = urllib.parse.urlencode(sorted_params, safe="")

        # Construct the base string

        base_string = f"{method.upper()}&{urllib.parse.quote(parsed_url.scheme + '://' + parsed_url.netloc + parsed_url.path, safe='')}&{urllib.parse.quote(param_str, safe='')}"

        # Create the signing key

        signing_key = f"{urllib.parse.quote(self.consumer_secret, safe='')}&{urllib.parse.quote(self.access_token_secret, safe='')}"

        # Generate the OAuth signature

        oauth_signature = base64.b64encode(

            hmac.new(signing_key.encode(), base_string.encode(), hashlib.sha256).digest()

        ).decode()

        # Add the signature to OAuth parameters

        oauth_params["oauth_signature"] = oauth_signature

        # Construct the Authorization header

        auth_header = "OAuth " + ", ".join(

            [f'{urllib.parse.quote(k)}="{urllib.parse.quote(v)}"' for k, v in oauth_params.items()]

        )

        return auth_header

    def _get_full_url(self, endpoint):

        """Construct the full URL for Magento API."""

        return f"https://{self.connection.host}{self.BASE_URL}/{endpoint}"

    def get_request(self, endpoint, search_criteria=None, method="GET", data=None):

        """Perform an API request to Magento."""

        if search_criteria:

            # Convert search criteria dictionary to URL parameters

            query_string = urlencode(search_criteria, doseq=True)

            endpoint = f"{endpoint}?{query_string}"

        url = self._get_full_url(endpoint)

        headers = {

            "Content-Type": "application/json",

            "Authorization": self._generate_oauth_parameters(url, method, data),

        }

        try:

            if method.upper() == "GET":

                response = requests.get(url, headers=headers, verify=False)

            else:

                response = requests.request(method, url, headers=headers, json=data)

            response.raise_for_status()  # Will raise an error for bad responses

        except requests.exceptions.HTTPError as http_err:

            error_details = None

            try:

                # Attempt to parse and log the error response

                error_details = response.json() if response.content else {}

                self.log.error("Error details: %s", error_details)

            except ValueError:

                # Failed to parse JSON response

                pass

            raise AirflowException(f"Request failed: {http_err}. Error details: {error_details}")

        except requests.exceptions.RequestException as e:

            raise AirflowException(f"Request failed: {str(e)}")

        return response.json()

    def post_request(self, endpoint, data=None):

        """Perform a POST API request to Magento."""

        url = self._get_full_url(endpoint)

        headers = {

            "Content-Type": "application/json",

            "Authorization": self._generate_oauth_parameters(url, "POST", data),

        }

        try:

            response = requests.post(url, headers=headers, json=data, verify=False)

            response.raise_for_status()  # Will raise an error for bad responses

        except requests.exceptions.HTTPError as http_err:

            error_details = None

            try:

                # Attempt to parse and log the error response

                error_details = response.json() if response.content else {}

                self.log.error("Error details: %s", error_details)

            except ValueError:

                # Failed to parse JSON response

                pass

            raise AirflowException(f"Request failed: {http_err}. Error details: {error_details}")

        except requests.exceptions.RequestException as e:

            raise AirflowException(f"Request failed: {str(e)}")

        return response.json()

`
ERROR - Error details: {'message': 'The signature is invalid. Verify and try again.', 'trace': "#0 /var/www/html/vendor/magento/framework/Oauth/Oauth.php(127): Magento\\Framework\\Oauth\\Oauth->_validateSignature(Array, 't1esssb0b0mb2kn...', 'POST', 'https://magento...', 'xsgorgioy3rea3s...')\n#1 /var/www/html/vendor/magento/module-webapi/Model/Authorization/OauthUserContext.php(78): Magento\\Framework\\Oauth\\Oauth->validateAccessTokenRequest(Array, 'https://magento...', 'POST')\n#2 /var/www/html/vendor/magento/module-authorization/Model/CompositeUserContext.php(84): Magento\\Webapi\\Model\\Authorization\\OauthUserContext->getUserId()\n#3 /var/www/html/vendor/magento/module-authorization/Model/CompositeUserContext.php(63): Magento\\Authorization\\Model\\CompositeUserContext->getUserContext()\n#4 /var/www/html/vendor/magento/module-webapi/Model/WebapiRoleLocator.php(45): Magento\\Authorization\\Model\\CompositeUserContext->getUserId()\n#5 /var/www/html/vendor/magento/framework/Authorization.php(47): Magento\\Webapi\\Model\\WebapiRoleLocator->getAclRoleId()\n#6 /var/www/html/vendor/magento/framework/Interception/Interceptor.php(58): Magento\\Framework\\Authorization->isAllowed('Magento_Custome...', NULL)\n#7 /var/www/html/vendor/magento/framework/Interception/Interceptor.php(138): Magento\\Framework\\Authorization\\Interceptor->___callParent('isAllowed', Array)\n#8 /var/www/html/vendor/magento/module-webapi/Model/Plugin/GuestAuthorization.php(38): Magento\\Framework\\Authorization\\Interceptor->Magento\\Framework\\Interception\\{closure}('Magento_Custome...', NULL)\n#9 /var/www/html/vendor/magento/framework/Interception/Interceptor.php(135): Magento\\Webapi\\Model\\Plugin\\GuestAuthorization->aroundIsAllowed(Object(Magento\\Framework\\Authorization\\Interceptor), Object(Closure), 'Magento_Custome...', NULL)\n#10 /var/www/html/vendor/magento/module-customer/Model/Plugin/CustomerAuthorization.php(55): Magento\\Framework\\Authorization\\Interceptor->Magento\\Framework\\Interception\\{closure}('Magento_Custome...', NULL)\n#11 /var/www/html/vendor/magento/framework/Interception/Interceptor.php(135): Magento\\Customer\\Model\\Plugin\\CustomerAuthorization->aroundIsAllowed(Object(Magento\\Framework\\Authorization\\Interceptor), Object(Closure), 'Magento_Custome...')\n#12 /var/www/html/vendor/magento/framework/Interception/Interceptor.php(153): Magento\\Framework\\Authorization\\Interceptor->Magento\\Framework\\Interception\\{closure}('Magento_Custome...')\n#13 /var/www/html/generated/code/Magento/Framework/Authorization/Interceptor.php(23): Magento\\Framework\\Authorization\\Interceptor->___callPlugins('isAllowed', Array, NULL)\n#14 /var/www/html/vendor/magento/module-customer/Model/AccountManagement.php(886): Magento\\Framework\\Authorization\\Interceptor->isAllowed('Magento_Custome...')

kodurusivakumar34 commented 1 month ago

It is working with requests_oauthlib lib.

https://github.com/sivajik34/magento-airflow/blob/main/src/apache_airflow_provider_magento/hooks/magento.py closing the issue

engcom-Hotel commented 4 weeks ago

Hello @sivajik34,

It seems the issue has been resolved at your end. Hence closing this issue.

Thanks

magento / magento2