benjaminstokes
commented
4 years ago Checkmarx version is 8.9 HF 0. This rather long script is what I used to generate the test data set i'm working with
# Create projects with the Checkmarx CLI
$cxUser="checkmarx"
$cxPass="redacted"
$cxServer="localhost"
[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072;
$progressPreference = 'silentlyContinue'
# Install the Checkmarx CLI
md -force c:\programdata\checkmarx\cli
$cli_url="https://download.checkmarx.com/9.0.0/Plugins/CxConsolePlugin-2020.2.18.zip"
$cli_zip=$cli_url.Substring($cli_url.LastIndexOf("/") + 1)
invoke-webrequest -uri "${cli_url}" -outfile "C:\programdata\checkmarx\cli\${cli_zip}"
cd "c:\programdata\checkmarx\cli\"
Expand-Archive "c:\programdata\checkmarx\cli\${cli_zip}" -DestinationPath "c:\programdata\checkmarx\cli\"
cd c:\programdata\checkmarx\cli\
Class CxArmPolicyManagementApiClient {
[string] $username
[string] $password
[string] $url
[string] $token
[string] $apiscope = "sast_rest_api cxarm_api"
[string] $clientid = "resource_owner_client"
[string] $clientsecret = "014DF517-39D1-4453-B7B3-9930C563627C"
CxArmPolicyManagementApiClient([string] $username, [string] $password, [string] $url) {
$this.username = $username
$this.password = $password
$this.url = $url
}
Login() {
$body = @{
username = $this.username
password = $this.password
grant_type = "password"
scope = $this.apiscope
client_id = $this.clientid
client_secret = $this.clientsecret
}
try {
$response = Invoke-RestMethod -uri "$($this.url)/cxrestapi/auth/identity/connect/token" -method post -body $body -contenttype 'application/x-www-form-urlencoded' -UseBasicParsing
} catch {
Write-Host "StatusCode:" $_.Exception.Response.StatusCode.value__
Write-Host "StatusDescription:" $_.Exception.Response.StatusDescription
throw "Cannot Get OAuth2 Token"
}
$this.token = "$($response.token_type) $($response.access_token)"
}
[PSObject] GET([string] $url) {
$headers = @{
Authorization = $this.token
Accept = "application/json;v=1.0"
}
try {
$response = Invoke-RestMethod -uri "$($this.url)${url}" -method get -headers $headers -UseBasicParsing
return $response
} catch {
Write-Host "StatusCode:" $_.Exception.Response.StatusCode.value__
Write-Host "StatusDescription:" $_.Exception.Response.StatusDescription
throw "An error has occured on GET $url"
}
}
[PSObject] POST([string] $url, $body) {
$body_json = ($body | ConvertTo-Json -Depth 10)
$headers = @{
Authorization = $this.token
}
try {
$response = Invoke-RestMethod -uri "$($this.url)${url}" -method post -headers $headers -UseBasicParsing -Body $body_json -ContentType "application/json;v=1.0"
return $response
} catch {
Write-Host "StatusCode:" $_.Exception.Response.StatusCode.value__
Write-Host "StatusDescription:" $_.Exception.Response.StatusDescription
throw "An error has occured on POST $url with $body_json"
}
}
[PSObject] PUTliteral([string] $url, $body) {
$body_json = $body
$headers = @{
Authorization = $this.token
}
try {
$response = Invoke-RestMethod -uri "$($this.url)${url}" -method put -headers $headers -UseBasicParsing -Body $body_json -ContentType "application/json;v=1.0"
return $response
} catch {
Write-Host "StatusCode:" $_.Exception.Response.StatusCode.value__
Write-Host "StatusDescription:" $_.Exception.Response.StatusDescription
throw "An error has occured on PUT $url with $body_json"
}
}
[PSObject] PUT([string] $url, $body) {
$body_json = ($body | ConvertTo-Json -Depth 10)
$headers = @{
Authorization = $this.token
}
try {
$response = Invoke-RestMethod -uri "$($this.url)${url}" -method put -headers $headers -UseBasicParsing -Body $body_json -ContentType "application/json;v=1.0"
return $response
} catch {
Write-Host "StatusCode:" $_.Exception.Response.StatusCode.value__
Write-Host "StatusDescription:" $_.Exception.Response.StatusDescription
throw "An error has occured on PUT $url with $body_json"
}
}
<######################
# Projects
#######################>
[PSObject] GetProjects() {
return $this.GET("/cxarm/policymanager/projects")
}
[PSObject] GetPoliciesByProjectId($id) {
return $this.GET("/cxarm/policymanager/projects/${id}/policies")
}
[PSObject] GetViolationsByProjectId($id) {
return $this.GET("/cxarm/policymanager/projects/${id}/violations?provider=SAST")
}
[PSObject] UpdatePoliciesByProjectId ($projectId, $policyId) {
$policies = "[ { `"policyId`": $policyId } ]"
return $this.PUTliteral("/cxarm/policymanager/projects/${projectId}/policies", $policies)
}
<######################
# Policies
#######################>
[PSObject] GetPolicies() {
return $this.GET("/cxarm/policymanager/policies")
}
[PSObject] FindPolicyIdByName([string] $name) {
$policies = $this.GetPolicies()
$policyId = $policies | where { $_.name -eq $name } | select -ExpandProperty id
return $policyId
}
[PSObject] GetPoliciesById($id) {
return $this.GET("/cxarm/policymanager/policies/${id}")
}
[PSObject] GetAssignedProjectsByPoliciyId($id) {
return $this.GET("/cxarm/policymanager/policies/${id}/projects")
}
[PSObject] CreatePolicy ($policy) {
return $this.POST("/cxarm/policymanager/policies", $policy)
}
<######################
# Rules
#######################>
[PSObject] GetRulesByPolicyId($id) {
return $this.GET("/cxarm/policymanager/policies/${id}/rules")
}
<######################
# SAST - Projects
#######################>
[PSObject] GetSastProjects() {
return $this.GET("/cxrestapi/projects")
}
[PSObject] FindProjectByName([string] $name) {
$projects = $this.GetSastProjects()
$projectId = $projects | where { $_.name -eq $name } | select -ExpandProperty id
return $projectId
}
[PSObject] CreateProject($project) {
return $this.POST("/cxrestapi/projects", $project)
}
[PSObject] UpdateProject($project, $id) {
return $this.PUT("/cxrestapi/projects/${id}", $project)
}
<######################
# SAST - Teams
#######################>
[PSObject] GetTeams() {
return $this.GET("/cxrestapi/auth/teams")
}
[string] FindTeamIdByName([string] $teamName) {
$teams = $this.GetTeams()
$teamid = $teams | where { $_.fullname -eq $teamName } | select -ExpandProperty id
return $teamid
}
<######################
# SAST - Custom Fields
#######################>
[PSObject] GetCustomFields() {
return $this.GET("/cxrestapi/customFields")
}
<######################
# SAST - Presets
#######################>
[PSObject] GetPresets() {
return $this.GET("/cxrestapi/sast/presets")
}
[PSObject] GetPresetById($id) {
return $this.GET("/cxrestapi/sast/presets/${id}")
}
[PSObject] FindPresetIdByName([string] $name) {
$presets = $this.GetPresets()
$presetId = $presets | where { $_.name -eq $name } | select -ExpandProperty id
return $presetId
}
<######################
# SAST - Engine Configurations
#######################>
[PSObject] GetEngineConfigurations() {
return $this.GET("/cxrestapi/sast/engineConfigurations")
}
[PSObject] GetEngineConfigurationById($id) {
return $this.GET("/cxrestapi/sast/engineConfigurations/${id}")
}
[PSObject] FindEngineConfigurationIdByName([string] $name) {
$engineConfigurations = $this.GetEngineConfigurations()
$engineConfigurationId = $engineConfigurations | where { $_.name -eq $name } | select -ExpandProperty id
return $engineConfigurationId
}
<######################
# SAST - Scans
#######################>
[PSObject] CreateScanSettings($settings) {
return $this.POST("/cxrestapi/sast/scanSettings", $settings)
}
}
Class CxWebServiceClient {
[string] $username
[string] $password
[string] $url
[string] $SessionId
$proxy
CxWebServiceClient([string] $username, [string] $password, [string] $url) {
$this.username = $username
$this.password = $password
$this.url = $url
$this.proxy = New-WebServiceProxy -Uri "$($this.url)/CxWebInterface/Portal/CxWebService.asmx?wsdl"
}
Login() {
$proxyType = $this.proxy.GetType().Namespace
$credentials = new-object ("$proxyType.Credentials")
$credentials.User = $this.username
$credentials.Pass = $this.password
$res = $this.proxy.Login($credentials, 1033)
$this.SessionId = $res.SessionId
}
[PSObject] SaveCustomFields($fields) {
$customfields = @()
$fields | ForEach-Object {
$proxyType = $this.proxy.GetType().Namespace
$customfield = New-Object ("$proxyType.CxWSCustomField")
$customfield.Id = 0 # Not a mistake, really zero
$customfield.Name = $_
$customfields+=$customfield
}
$res = $this.proxy.SaveCustomFields($this.sessionId, $customfields)
if($res.IsSuccesfull){
Write-Host "Custom fields have been created" -ForegroundColor Green
} else{
Write-Host $res.ErrorMessage -ForegroundColor Red
}
return $res
}
[PSObject] CreateServiceProvider($name) {
$res = $this.proxy.CreateNewServiceProvider($this.SessionId, $name, 1, 1, 1, 1, $null)
if ($res.IsSuccesfull) {
Write-Host "Created Service Provider" -ForegroundColor Green
} else {
Write-Host "Error creating service provider" -ForegroundColor Red
}
return $res
}
[PSObject] CreateCompany($name, $parentId) {
$res = $this.proxy.CreateNewCompany($this.SessionId, $parentId, $name, 1, 1, 1, 1, $null)
if ($res.IsSuccesfull) {
Write-Host "Created Company" -ForegroundColor Green
} else {
Write-Host "Error creating company" -ForegroundColor Red
}
return $res
}
[PSObject] CreateTeam($name, $parentId) {
$res = $this.proxy.CreateNewTeam($this.SessionId, $parentId, $name, $null)
if ($res.IsSuccesfull) {
Write-Host "Created Team" -ForegroundColor Green
} else {
Write-Host "Error creating team" -ForegroundColor Red
}
return $res
}
}
$projects = @(
"https://github.com/appsecco/dvna",
"https://github.com/WebGoat/WebGoat",
"https://github.com/OWASP/NodeGoat"
"https://github.com/naudio/NAudio",
"https://github.com/nopSolutions/nopCommerce",
"https://github.com/lukencode/FluentEmail",
"https://github.com/ethicalhack3r/DVWA",
"https://github.com/notepad-plus-plus/notepad-plus-plus",
"https://github.com/appsecco/dvja",
"https://github.com/CSPF-Founder/JavaVulnerableLab",
"https://github.com/nvisium-jack-mannino/OWASP-GoatDroid-Project",
"https://github.com/interference-security/DVWS",
"https://github.com/snoopysecurity/dvws-node",
"https://github.com/htbridge/pivaa",
"https://github.com/payatu/diva-android",
"https://github.com/CSPF-Founder/DodoVulnerableBank",
"https://github.com/dineshshetty/Android-InsecureBankv2",
"https://github.com/dan7800/VulnerableAndroidAppOracle",
"https://github.com/CyberScions/Digitalbank",
"https://github.com/logicalhacking/DVHMA",
"https://github.com/bkimminich/juice-shop",
"https://github.com/prateek147/DVIA",
"https://github.com/prateek147/DVIA-v2",
"https://github.com/hclproducts/AltoroJ"
)
[CxArmPolicyManagementApiClient] $cxrest = [CxArmPolicyManagementApiClient]::New($cxUser, $cxPass, $cxServer)
$cxrest.Login()
[CxWebServiceClient] $cxWs = [CxWebServiceClient]::New($cxUser, $cxPass, "http://${cxServer}")
$cxWs.Login()
foreach ($url in $projects) {
$projectName = $url.Substring($url.LastIndexOf("/") + 1)
$CompanyId = $cxrest.FindTeamIdByName("\CxServer\serviceprovider\company\")
$existingTeam = $cxrest.FindTeamIdByName("\CxServer\serviceprovider\company\${projectName}\")
if ($null -ne $existingTeam) {
Write-Host "SKIPPED: $team already exists" -ForegroundColor Yellow
} else {
Write-Host "Created \CxServer\serviceprovider\company\${projectName}\" -ForegroundColor Green
$res = $cxWs.CreateTeam("\CxServer\serviceprovider\company\${projectName}\", $CompanyId)
}
}
foreach ($url in $projects) {
$projectName = $url.Substring($url.LastIndexOf("/") + 1)
.\runCxConsole.cmd AsyncScan -v -projectName "CxServer\serviceprovider\company\${projectName}\${projectName}_sandbox" -CxServer "${cxServer}" -CxUser ${cxUser} -CxPassword ${cxPass} -LocationType GIT -LocationURL "$url" -LocationBranch "refs/heads/master" -ForceScan
}
foreach ($url in $projects) {
$projectName = $url.Substring($url.LastIndexOf("/") + 1)
.\runCxConsole.cmd AsyncScan -v -projectName "CxServer\serviceprovider\company\${projectName}_policy" -CxServer "${cxServer}" -CxUser ${cxUser} -CxPassword ${cxPass} -LocationType GIT -LocationURL "$url" -LocationBranch "refs/heads/master" -ForceScan
}
nleach999
Description
When running CxAnalytix with lots of scan data to process (in this case 52 projects with 48 scans) an exception can occur that causes all data to not be extracted due to a
System.Net.Sockets.SocketException: Only one usage of each socket address (protocol/network address/port) is normally permitted.Expected Behavior
No exceptions should occur regardless of how many projects or scans are being extracted in a single run or lifetime of the CxAnalytix process.
Actual Behavior
Reproduction
CxAnalytix 1.1.3 on Windows Server 2016. Starting with a clean State File and needing to extract data for 52 projects and 48 scans. Configuration is set for MongoDB:
Build the tool as per the tutorial and run the command line. Notice that the exception (above) occurs and you do not have all of your data loaded into MongoDB.
Note:
Initial research indicates this is a known issue with HttpClient and the solution is instantiate it once (singleton) and use many times.
This issue report from Azure Functions seems to be relevant to this bug report in CxAnalytix: https://github.com/Azure/azure-functions-host/issues/1806
Environment Details
CxAnalytix v1.1.3 on Server 2016.