[Question] Linking browserstack / jira to data driven tests

[jensencp7]

Hi Cen, Hope you are well.

Consider the following , these are data driven tests that are created at run time from either a json object or csv file. each array / record ends up being a test. In the report the csv tests = 500... How do i link test case id's to each test or a jira ticket number

Currently we have to explicitly add this annotation at the start of each test function , it works well for single / stand alone tests but not for data driven tests ...Solution for this ? console.log inside the test function ? or add in the test title ? Need a less painful way to manage this.

  /**
   * Add extra information for the case
   * @owner Jensen
   * @jira MCR-16889
   */

  const testData = [
    { name: 'Luffy', job: 'Pirate', testnum: 'Test 5: ' },
    { name: 'Zoro', job: 'Swordsman', testnum: 'Test 6: ' },
    { name: 'Sanji', job: 'Chef', testnum: 'Test 7: ' }
  ];

  test.describe('Data Driven Tests JSON', () => {
    testData.forEach((data) => {
      test(`${data.testnum} Example Data Driven Test from JSON`, async ({ request }) => {
        // Make a POST request with the test data
        const response = await request.post('https://reqres.in/api/users', { data });
        expect(response.ok()).toBeTruthy();

        // Parse the response
        const user = await response.json();
        console.log("API response:", user);

        // Validate the response
        expect(user.name).toBe(data.name);
      });
    });
  });

// Read and parse the CSV file
const records = parse(fs.readFileSync(path.join(__dirname, 'test-data-example.csv')), {
    columns: true,
    //from : 1, //start from row
    //to : 4, //end at row 2
    skip_empty_lines: true
  });

  test.describe('Data Driven Tests CSV', () => {
    for (const record of records) {
      test(`${record.test_case} : Example Data Driven Test From CSV`, { tag: ['@Positive'] }, async () => {
        // Log the record details
        console.log(`Test number: ${record.test_case}`);
        console.log(`CIF number: ${record.cif_number}`);
        console.log(`Account number: ${record.account_number}`);
        console.log(`Total: ${record.total}`);

        // Additional test logic can be added here if necessary
      });
    }
  });

Here is the browserstack.js code that you wrote , i got it working to create test runs dynamically

import EC from 'eight-colors';
import dotenv from 'dotenv';
import axios from 'axios';

// https://github.com/motdotla/dotenv
dotenv.config();

export default async (reportData, helper) => {

    const akiKey = process.env.BROWSERSTACK_API_KEY;
    const client = createClient(akiKey);

    // api doc
    // https://www.browserstack.com/docs/test-management/api-reference/test-runs

    const project_id = 'PR-8'; //found on the projects overview page
    const allCases = helper.filter((item) => item.type === 'case' && item.browserstack);

    // Get test runs list
    //const runs = await client.get(`/api/v2/projects/${project_id}/test-runs`);
    //console.log(runs);

    // Step 1, Create test run
    const res = await client.post(`/api/v2/projects/${project_id}/test-runs`, {
        test_run: {
            name: "Test Run-29/11/2023",
            description: "check the performance of the test run",
            run_state: "new_run",
            configurations: [],
            test_cases: [],
            folder_ids: [],
            include_all: true
        }
    });

    if (!res || !res.success) {
        return;
    }

    const testrun = res.test_run;
    //console.log(testrun);

    const test_run_id = res.test_run.identifier;
    //console.log(test_run_id);

    // Step 2: Add test result to test run
    const allRequest = allCases.map((item) => {
        return client.post(`/api/v2/projects/${project_id}/test-runs/${test_run_id}/results`, {
            test_case_id: item.browserstack,
            test_result: {
                status: item.caseType  // passed, failed
            }
        });
    });

    let err;
    await Promise.all(allRequest).catch((e) => {
        err = e;
    });

    if (err) {
        console.log(err);
        return;
    }

    // Step 3: Close test run

    EC.logGreen('[browserstack] completed');

};

const createClient = function(akiKey) {
    const entrypoint = 'https://test-management.browserstack.com';
    const headers = {
        Authorization: `Basic ${Buffer.from(akiKey).toString('base64')}`
    };
    // console.log(headers);
    const handleError = (err, api) => {
        EC.logRed(EC.red(`ERROR: Catch a error on: ${api}`));
        if (err.response) {
            console.log(err.response.data);
        } else if (err.request) {
            console.log(err.request);
        } else {
            console.log(err.message);
        }
    };

    return {
        get: async function(api) {
            let err;
            const res = await axios.get(entrypoint + api, {
                headers
            }).catch(function(e) {
                err = e;
            });
            if (err) {
                handleError(err, api);
                return;
            }
            return res.data;
        },
        post: async function(api, data) {
            // console.log(data);
            let err;
            const res = await axios.post(entrypoint + api, data, {
                headers
            }).catch(function(e) {
                err = e;
            });
            if (err) {
                handleError(err, api);
                return;
            }
            return res.data;
        }
    };
};

[jensencp7]

Each iteration will require a Test case id value , but the value in annotation is static .. so now i can upload test results for these

test.describe('Data Driven Tests CSV', () => {
    for (const record of records) {
        /**
        * Add extra information for the case
        * @owner Jensen
        * @jira MCR-16889
        * @browserstack TC-122
        */  
      test(`${record.test_case} : Example Data Driven Test From CSV`, { tag: ['@Positive'] }, async () => {
        // Log the record details
        console.log(`Test number: ${record.test_case}`);
        console.log(`CIF number: ${record.cif_number}`);
        console.log(`Account number: ${record.account_number}`);
        console.log(`Total: ${record.total}`);

        // Additional test logic can be added here if necessary
      });
    }
  });

[cenfun]

It is a featrue request. Let's figure out how to fix it.

[cenfun]

Please try new version 2.6.0 https://github.com/cenfun/monocart-reporter/releases/tag/2.6.0

[jensencp7]

Hi Cen, thanks for the update. Set Meta Data works well. See below usage.

Ive updated the browserstack.js code to update results via batch instead of single. Too many single api calls take too long and times out, just better to do via batch. See below example and test.

import EC from 'eight-colors';
import dotenv from 'dotenv';
import axios from 'axios';

// https://github.com/motdotla/dotenv
dotenv.config();
const now = new Date();
const project_name = "monocart";
const project_desc = "Regression tests for monocart browserstack";
const project_id = 'PR-8'; //can be found on the projects overview page https://test-management.browserstack.com/

export default async (reportData, helper) => {

    const apiKey = process.env.BROWSERSTACK_API_KEY;
    const client = createClient(apiKey);

    // api doc
    // https://www.browserstack.com/docs/test-management/api-reference/test-runs

    const allCases = helper.filter((item) => item.type === 'case' && item.browserstack);

    // Get test runs list
    //const runs = await client.get(`/api/v2/projects/${project_id}/test-runs`);
    //console.log(runs);

    // Step 1, Create test run
    const res = await client.post(`/api/v2/projects/${project_id}/test-runs`, {
        test_run: {
            name: `${project_name} Regression Test Run ${now}`,
            description: "Execute the examples tests provided in the template",
            run_state: "new_run",
            configurations: [],
            test_plan_id : "TP-3",
            test_cases: [],
            folder_ids: [],
            include_all: true
        }
    });

    if (!res || !res.success) {
        return;
    }

    const testrun = res.test_run;
    //console.log(testrun);

    const test_run_id = res.test_run.identifier;
    //console.log(test_run_id);

    // Step 2: Add test result to test run
    const allRequest = allCases.map((item) => {
        //console.log(`Making API request for test_case_id: ${item.browserstack}, status: ${item.caseType}`);

        return {
            test_case_id: item.browserstack,
            configuration_id: item.configurationId, // Assuming you have a configuration ID for each test case
            test_result: {
                status: item.caseType,
                description: "NA",
                issues: [
                    "TR-2",
                    "XYZ-2"
                ],
                custom_fields: {
                    custom_fields_name: "value"
                }
            }
        };
    });

    // Send the structured data in one API request
    try {
        const response = await client.post(`/api/v2/projects/${project_id}/test-runs/${test_run_id}/results`, {
            results: allRequest
        });

        //console.log("API request successful");
    } catch (error) {
        console.error("Error making API request:", error);
    }

    EC.logGreen('[browserstack] results uploaded successfully.');

    let err;
    await Promise.all(allRequest).catch((e) => {
        err = e;
    });

    if (err) {
        console.log(err);
        return;
    }

    // Step 3: Close test run

    //EC.logGreen('[browserstack] completed');

};

const createClient = function(apiKey) {
    const entrypoint = 'https://test-management.browserstack.com';
    const headers = {
        Authorization: `Basic ${Buffer.from(apiKey).toString('base64')}`
    };
    // console.log(headers);
    const handleError = (err, api) => {
        EC.logRed(EC.red(`ERROR: Catch a error on: ${api}`));
        if (err.response) {
            console.log(err.response.data);
        } else if (err.request) {
            console.log(err.request);
        } else {
            console.log(err.message);
        }
    };

    return {
        get: async function(api) {
            let err;
            const res = await axios.get(entrypoint + api, {
                headers
            }).catch(function(e) {
                err = e;
            });
            if (err) {
                handleError(err, api);
                return;
            }
            return res.data;
        },
        post: async function(api, data) {
            // console.log(data);
            let err;
            const res = await axios.post(entrypoint + api, data, {
                headers
            }).catch(function(e) {
                err = e;
            });
            if (err) {
                handleError(err, api);
                return;
            }
            return res.data;
        }
    };
};

const { test, expect } = require('@playwright/test');
const { Client } = require('pg');
require('dotenv').config();
const fs = require('fs');
const path = require('path');
const { parse } = require('csv-parse/sync');
const { setMetadata } = require('monocart-reporter');

const testData = [
    { name: 'Luffy', job: 'Pirate', testnum: 'Test 5: ', owner: 'Jensen', browserstack: 'TC-226' },
    { name: 'Zoro', job: 'Swordsman', testnum: 'Test 6: ', owner: 'Jensen', browserstack: 'TC-227' },
    { name: 'Sanji', job: 'Chef', testnum: 'Test 7: ', owner: 'Jensen', browserstack: 'TC-228' }
];

test.describe('Data Driven Tests JSON', () => {
    testData.forEach((data) => {
        test(`${data.testnum} Example Data Driven Test from JSON`, async ({ request }) => {
            setMetadata({ owner: data.owner, browserstack: data.browserstack }, test.info());

            // Make a POST request with the test data
            const response = await request.post('https://reqres.in/api/users', { data });
            expect(response.ok()).toBeTruthy();

            // Parse the response
            const user = await response.json();
            console.log("API response:", user);

            // Validate the response
            expect(user.name).toBe(data.name);
        });
    });
});

// Read and parse the CSV file
const records = parse(fs.readFileSync(path.join(__dirname, 'test-data-example.csv')), {
    columns: true,
    delimiter: ';',
    //from : 1, //start from row
    //to : 4, //end at row 
    skip_empty_lines: true
});

test.describe('Data Driven Tests CSV', () => {
    for (const record of records) {

        test(`${record.test_case} : Example Data Driven Test From CSV`, async () => {
           setMetadata({ owner: record.owner, browserstack: record.browserstack , jira : record.jira }, test.info());
            // Log the record details
            console.log(`Test number: ${record.test_case}`);
            console.log(`CIF number: ${record.cif_number}`);
            console.log(`Account number: ${record.account_number}`);
            console.log(`Total: ${record.total}`);

            if(record.test_case == 'Test 1' || record.test_case == 'Test 10')
            {
                test.fail();
            }

            // Additional test logic can be added here if necessary
        });
    }
});

[jensencp7]

I picked up one issue , i cant set tags in setMetaData

const testData = [
    { name: 'Luffy', job: 'Pirate', testnum: 'Test 5: ', owner: 'Jensen', browserstack: 'TC-226',tag : '@Positive' },
    { name: 'Zoro', job: 'Swordsman', testnum: 'Test 6: ', owner: 'Jensen', browserstack: 'TC-227',tag : 'Positive' },
    { name: 'Sanji', job: 'Chef', testnum: 'Test 7: ', owner: 'Jensen', browserstack: 'TC-228',tag : 'Positive' }
];

test.describe('Data Driven Tests JSON', () => {
    testData.forEach((data) => {
        test.only(`${data.testnum} Example Data Driven Test from JSON`, async ({ request }) => {
            setMetadata({ owner: data.owner, browserstack: data.browserstack , tag:data.tag}, test.info());

            // Make a POST request with the test data
            const response = await request.post('https://reqres.in/api/users', { data });
            expect(response.ok()).toBeTruthy();

            // Parse the response
            const user = await response.json();
            console.log("API response:", user);

            // Validate the response
            expect(user.name).toBe(data.name);
        });
    });
});

// @ts-check
const { defineConfig } = require('@playwright/test');
import browserstack from './browserstack.js';

/**
 * Read environment variables from file.
 * https://github.com/motdotla/dotenv
 */
// require('dotenv').config();

/**
 * @see https://playwright.dev/docs/test-configuration
 */
module.exports = defineConfig({
  testDir: './tests',
  /* Run tests in files in parallel */
  fullyParallel: true,
  /* Fail the build on CI if you accidentally left test.only in the source code. */
  forbidOnly: !!process.env.CI,
  /* Retry on CI only */
  retries: process.env.CI ? 2 : 0,
  /* Opt out of parallel tests on CI. */
  workers: 10,
  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
  reporter: [['line'], ['monocart-reporter', {
    name: "Playwright Test Report",
    outputFile: 'test-results/report.html',
    onEnd: async (reportData, helper) => {

      //await browserstack(reportData, helper);

    },
    tags: {
      'Positive': {
        style: {
          background: '#008000'
        },
        description: 'This is a positive test'
      },
      'Negative': {
        style: {
          background: '#d00'
        },
        description: 'This is a negative test'
      }
    },

    // custom columns
    columns: (defaultColumns) => {

      // disable title tags
      defaultColumns.find((column) => column.id === 'title').titleTagsDisabled = true;

      // add tags column
      const index = defaultColumns.findIndex((column) => column.id === 'type');
      defaultColumns.splice(index, 0, {
        // define the column in reporter
        id: 'owner',
        name: 'Owner',
        align: 'center',
        searchable: true,
        styleMap: {
          'font-weight': 'normal'
        }
      },
        {
          // another column for JIRA link
          id: 'jira',
          name: 'JIRA #',
          width: 100,
          searchable: true,
          styleMap: 'font-weight:normal;',
          formatter: (v, rowItem, columnItem) => {
            const key = rowItem[columnItem.id];
            return `<a href="https://your-jira-url/${key}" target="_blank">${v}</a>`;
          },
        },
        {
          id: 'tags',
          name: 'Tags',
          width: 150,
          formatter: 'tags'
        }

      );

    }
  }]],
  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
  use: {
    ignoreHTTPSErrors: true,
  /* Base URL to use in actions like `await page.goto('/')`. */
  // baseURL: 'http://127.0.0.1:3000',

  /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
  //trace: 'on-first-retry',
  },

  /* Configure projects for major browsers */
  //projects: [
  // {
  //   name: 'chromium',
  //   use: { ...devices['Desktop Chrome'] },
  // },

  // {
  //   name: 'firefox',
  //   use: { ...devices['Desktop Firefox'] },
  // },

  // {
  //   name: 'webkit',
  //   use: { ...devices['Desktop Safari'] },
  // },

  /* Test against mobile viewports. */
  // {
  //   name: 'Mobile Chrome',
  //   use: { ...devices['Pixel 5'] },
  // },
  // {
  //   name: 'Mobile Safari',
  //   use: { ...devices['iPhone 12'] },
  // },

  /* Test against branded browsers. */
  // {
  //   name: 'Microsoft Edge',
  //   use: { ...devices['Desktop Edge'], channel: 'msedge' },
  // },
  // {
  //   name: 'Google Chrome',
  //   use: { ...devices['Desktop Chrome'], channel: 'chrome' },
  // },
  //],

  /* Run your local dev server before starting the tests */
  // webServer: {
  //   command: 'npm run start',
  //   url: 'http://127.0.0.1:3000',
  //   reuseExistingServer: !process.env.CI,
  // },
});

[cenfun]

We can use new syntax to tag test, see https://playwright.dev/docs/test-annotations#tag-tests

test(item.title, {
    tag: item.tag
}, () => {

    setMetadata({
        owner: item.owner,
        jira: item.jira
    }, test.info());

});

[jensencp7]

Works perfectly , thanks ! Please close this issue if you are happy. Appreciate the feedback and help, my favorite reporter !

const testData = [
    { name: 'Luffy', job: 'Pirate', testnum: 'Test 5: ', owner: 'Jensen', browserstack: 'TC-226',tag : '@Positive' },
    { name: 'Zoro', job: 'Swordsman', testnum: 'Test 6: ', owner: 'Jensen', browserstack: 'TC-227',tag : '@Positive' },
    { name: 'Sanji', job: 'Chef', testnum: 'Test 7: ', owner: 'Jensen', browserstack: 'TC-228',tag : '@Positive' }
];

test.describe('Data Driven Tests JSON', () => {
    testData.forEach((data) => {
        test(`${data.testnum} Example Data Driven Test from JSON`,{tag: data.tag} ,async ({ request }) => {
            setMetadata({ owner: data.owner, browserstack: data.browserstack}, test.info());

            // Make a POST request with the test data
            const response = await request.post('https://reqres.in/api/users', { data });
            expect(response.ok()).toBeTruthy();

            // Parse the response
            const user = await response.json();
            console.log("API response:", user);

            // Validate the response
            expect(user.name).toBe(data.name);
        });
    });
});