Closed ehennum closed 4 years ago
georgeajit
commented
6 years ago Ran the tests (QA functional tests without DMSDK) with develop branch. Had about 150 test failures. Have started running with 4.1.1-develop branch. Will post results and findings.
georgeajit
commented
6 years ago https://project.marklogic.com/jira/browse/DHFPROD-1382 has the test details. On the DHS stack, created "dhaas-mod-writer" who has endpointDeveloper role assigned. Create another user with ("rest-admin" user) who has endpointUser role assigned.
Test to insert modules into the modules database using Java Client API DocumentManager
package com.marklogic.client.functionaltest;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.util.stream.Stream;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import com.fasterxml.jackson.databind.node.ArrayNode;
import org.xml.sax.InputSource;
import com.marklogic.client.DatabaseClient;
import com.marklogic.client.DatabaseClientFactory;
import com.marklogic.client.DatabaseClientFactory.Authentication;
import com.marklogic.client.FailedRequestException;
import com.marklogic.client.ForbiddenUserException;
import com.marklogic.client.SessionState;
import com.marklogic.client.document.JSONDocumentManager;
import com.marklogic.client.document.TextDocumentManager;
import com.marklogic.client.impl.BaseProxy;
import com.marklogic.client.io.DocumentMetadataHandle;
import com.marklogic.client.io.DocumentMetadataHandle.Capability;
import com.marklogic.client.io.FileHandle;
import com.marklogic.client.io.Format;
import com.marklogic.client.io.JacksonHandle;
import com.marklogic.client.DatabaseClient.ConnectionType;
import com.marklogic.client.DatabaseClientFactory;
import com.marklogic.client.io.StringHandle;
public class ClientApiDHaaSModuleInsertTest extends BasicJavaClientRESTDHaaS {
private static String host = null;
private static int modulesPort = 8004;
private static DatabaseClient schemaDBclient = null;
// Create an identifier for modules document - Client API call will be to endpoint
private static String endPointURI_1 = "/ext/TestE2EIntegerParamReturnDouble/TestE2EIntegerParamReturnDouble";
private static String endPointURI_2 = "/ext/TestE2EIntegerParamReturnDouble/TestE2EIntegerParamReturnDoubleErrorCond";
private static String endPointURI_3 = "/ext/TestE2EIntegerParamReturnDouble/TestRequiredParam";
private static String endPointURI_5 = "/ext/TestE2EMultiStringsInStringsOut/TestE2EJsonStringsInStringsOut";
private static String endPointURI_6 = "/ext/TestE2EModuleNotFound/TestE2EModuleNotFound";
private static String endPointURI_7 = "/ext/TestE2ESession/TestE2ESession";
@BeforeClass
public static void setUp() throws Exception {
System.out.println("In setup");
loadGradleProperties();
host = getServer();
// dhaas-mod-writer has endpointDeveloper role assigned.
schemaDBclient = DatabaseClientFactory.newClient(getRestServerHostName(), getRestServerPort(), new DatabaseClientFactory.DigestAuthContext("dhaas-mod-writer", "x"), ConnectionType.GATEWAY);
TextDocumentManager docMgr = schemaDBclient.newTextDocumentManager();
File file = new File(
"src/test/java/com/marklogic/client/functionaltest/data/api/TestE2EIntegerParamReturnDouble.sjs");
// create a handle on the content
FileHandle handle = new FileHandle(file);
DocumentMetadataHandle metadataHandle = new DocumentMetadataHandle();
metadataHandle.getPermissions().add("endpointUser", Capability.UPDATE, Capability.READ, Capability.EXECUTE);
// write the document content
docMgr.write(endPointURI_1+".sjs", metadataHandle, handle);
file = null;
handle = null;
file = new File(
"src/test/java/com/marklogic/client/functionaltest/data/api/TestE2EIntegerParamReturnDouble.api");
handle = new FileHandle(file);
docMgr.write(endPointURI_1+".api", metadataHandle, handle);
file = null;
handle = null;
file = new File(
"src/test/java/com/marklogic/client/functionaltest/data/api/TestE2EIntegerParamReturnDoubleErrorCond.sjs");
handle = new FileHandle(file);
docMgr.write(endPointURI_2+".sjs", metadataHandle, handle);
file = null;
handle = null;
file = new File(
"src/test/java/com/marklogic/client/functionaltest/data/api/TestE2EIntegerParamReturnDoubleErrorCond.api");
handle = new FileHandle(file);
docMgr.write(endPointURI_2+".api", metadataHandle, handle);
file = null;
handle = null;
file = new File("src/test/java/com/marklogic/client/functionaltest/data/api/TestRequiredParam.sjs");
handle = new FileHandle(file);
docMgr.write(endPointURI_3+".sjs", metadataHandle, handle);
file = null;
handle = null;
file = new File("src/test/java/com/marklogic/client/functionaltest/data/api/TestRequiredParam.api");
handle = new FileHandle(file);
docMgr.write(endPointURI_3+".api", metadataHandle, handle);
file = null;
handle = null;
file = new File("src/test/java/com/marklogic/client/functionaltest/data/api/TestE2EJsonStringsInStringsOut.sjs");
handle = new FileHandle(file);
docMgr.write(endPointURI_5+".sjs", metadataHandle, handle);
file = null;
handle = null;
file = new File("src/test/java/com/marklogic/client/functionaltest/data/api/TestE2EJsonStringsInStringsOut.api");
handle = new FileHandle(file);
docMgr.write(endPointURI_5+".api", metadataHandle, handle);
file = null;
handle = null;
// Do not add the SJS module to DB
file = new File("src/test/java/com/marklogic/client/functionaltest/data/api/TestE2EModuleNotFound.api");
handle = new FileHandle(file);
docMgr.write(endPointURI_6+".api", metadataHandle, handle);
file = null;
handle = null;
file = new File("src/test/java/com/marklogic/client/functionaltest/data/api/TestE2ESession.sjs");
handle = new FileHandle(file);
docMgr.write(endPointURI_7+".sjs", metadataHandle, handle);
file = null;
handle = null;
file = new File("src/test/java/com/marklogic/client/functionaltest/data/api/TestE2ESession.api");
handle = new FileHandle(file);
docMgr.write(endPointURI_7+".api", metadataHandle, handle);
file = null;
handle = null;
/*file = new File("src/test/java/com/marklogic/client/functionaltest/data/api/TestE2EJsonDocsInStringsOut.sjs");
handle = new FileHandle(file);
docMgr.write(endPointURI_4, metadataHandle, handle);
file = null;
handle = null;*/
schemaDBclient.release();
}
@Test
public void testModulesDB() throws Exception
{
System.out.println("Running testModulesDB");
schemaDBclient = DatabaseClientFactory.newClient(getRestServerHostName(), getRestServerPort(), new DatabaseClientFactory.DigestAuthContext("dhaas-mod-writer", "x"), ConnectionType.GATEWAY);
// Verify if the docs got inserted into the database.
StringHandle contentHandle = readDocumentUsingStringHandle(schemaDBclient, endPointURI_1+".sjs", "Text");
// get the contents
String readContent = contentHandle.get();
System.out.println(readContent);
}
@AfterClass
public static void tearDownAfterClass() throws Exception {
System.out.println("In tear down");
}
}Test with Client API invoke
package com.marklogic.client.functionaltest;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.util.stream.Stream;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.marklogic.client.DatabaseClient;
import com.marklogic.client.DatabaseClientFactory;
import com.marklogic.client.DatabaseClientFactory.Authentication;
import com.marklogic.client.FailedRequestException;
import com.marklogic.client.ForbiddenUserException;
import com.marklogic.client.SessionState;
import com.marklogic.client.document.JSONDocumentManager;
import com.marklogic.client.document.TextDocumentManager;
import com.marklogic.client.impl.BaseProxy;
import com.marklogic.client.io.DocumentMetadataHandle;
import com.marklogic.client.io.DocumentMetadataHandle.Capability;
import com.marklogic.client.io.FileHandle;
import com.marklogic.client.io.Format;
import com.marklogic.client.io.JacksonHandle;
import com.marklogic.client.DatabaseClient.ConnectionType;
public class ClientApiDHaaSE2ETest extends BasicJavaClientRESTDHaaS {
private static DatabaseClient dbclient = null;
// Create an identifier for modules document - Client API call will be to endpoint
private static String endPointURI_1 = "/ext/TestE2EIntegerParamReturnDouble/TestE2EIntegerParamReturnDouble";
private static String endPointURI_2 = "/ext/TestE2EIntegerParamReturnDouble/TestE2EIntegerParamReturnDoubleErrorCond";
private static String endPointURI_3 = "/ext/TestE2EIntegerParamReturnDouble/TestRequiredParam";
private static String endPointURI_5 = "/ext/TestE2EMultiStringsInStringsOut/TestE2EJsonStringsInStringsOut";
private static String endPointURI_6 = "/ext/TestE2EModuleNotFound/TestE2EModuleNotFound";
private static String endPointURI_7 = "/ext/TestE2ESession/TestE2ESession";
@BeforeClass
public static void setUp() throws Exception {
System.out.println("In setup");
loadGradleProperties();
String analyticsHost = "DHF-AJ-OAlb-EXQDAVXSYKKO-2085710470.us-east-1.elb.amazonaws.com";
dbclient = DatabaseClientFactory.newClient(analyticsHost, 8009, new DatabaseClientFactory.DigestAuthContext("rest-admin", "x"), ConnectionType.GATEWAY);
}
@AfterClass
public static void tearDownAfterClass() throws Exception {
System.out.println("In tear down");
// release client
dbclient.release();
}
@Test
public void TestE2EItemPrice() throws Exception {
System.out.println("Running TestE2EItemPrice");
// Invoke the function
Double responseBack1 = TestE2EIntegerParaReturnDouble.on(dbclient).TestE2EItemPrice(new Integer(0));
System.out.println("Expected 1.0. Response from the Client API call is " + responseBack1);
assertEquals(1.0, responseBack1, 0.00);
Double responseBack2 = TestE2EIntegerParaReturnDouble.on(dbclient).TestE2EItemPrice(new Integer(-1));
System.out.println("Expected 0.0. Response from the Client API call is " + responseBack2);
assertEquals(0.0, responseBack2, 0.00);
Double responseBack3 = TestE2EIntegerParaReturnDouble.on(dbclient).TestE2EItemPrice(new Integer(1));
System.out.println("Expected 2.0. Response from the Client API call is " + responseBack3);
assertEquals(2.0, responseBack3, 0.00);
Double responseBack4 = TestE2EIntegerParaReturnDouble.on(dbclient).TestE2EItemPrice(0);
System.out.println("Expected 1.0. Response from the Client API call is " + responseBack4);
assertEquals(1.0, responseBack4, 0.00);
short srt = 12;
Double responseBack5 = TestE2EIntegerParaReturnDouble.on(dbclient).TestE2EItemPrice((int) srt);
System.out.println("Expected 13.0. Response from the Client API call is " + responseBack5);
assertEquals(13.0, responseBack5, 0.00);
// Integer.MAX_VALUE
double responseBack6 = TestE2EIntegerParaReturnDouble.on(dbclient)
.TestE2EItemPrice(new Integer(Integer.MAX_VALUE));
System.out.println("Expected Integer.MAX_VALUE. Response from the Client API call is " + responseBack6);
assertTrue("Expected value not returned", String.valueOf(responseBack6).contains("2.147483648E9"));
// Integer.MIN_VALUE
double responseBack7 = TestE2EIntegerParaReturnDouble.on(dbclient)
.TestE2EItemPrice(new Integer(Integer.MIN_VALUE));
System.out.println("Expected Integer.MIN_VALUE. Response from the Client API call is " + responseBack7);
assertTrue("Expected value not returned", String.valueOf(responseBack7).contains("-2.147483647E9"));
// Expecting incorrect data type from module
double responseBack8 = 0.0;
try {
responseBack8 = TestE2EIntegerParaReturnDouble.on(dbclient).TestE2EItemPrice(new Integer(10));
} catch (Exception ex) {
System.out.println("Expecting 0. Response from the Client API call is " + responseBack8);
System.out.println("Exception response from the Client API call is " + ex.toString());
assertTrue("Exception message incorrect", ex.toString().contains("java.lang.IllegalArgumentException: Could not convert to double: String10"));
assertEquals(0.0, responseBack8, 0.00);
}
// Null input. Expect 55555.00 as the return from API module so that we know
// passing null works
Double responseBack9 = TestE2EIntegerParaReturnDouble.on(dbclient).TestE2EItemPrice(null);
System.out.println("Expected 55555.00. Response from the Client API call is " + responseBack9);
assertEquals(55555.00, responseBack9, 0.00);
// Verify calls with two parameters - both nulls
float responseBack11 = TestE2EIntegerParaReturnDouble.on(dbclient).TestE2EItemPriceErrorCond(null, null);
System.out.println("Expected 30000.0. Response from the Client API call is " + responseBack11);
assertEquals(35000.0, responseBack11, 0.00);
// Verify calls with two parameters - second null parameter
float responseBack12 = TestE2EIntegerParaReturnDouble.on(dbclient).TestE2EItemPriceErrorCond(10, null);
System.out.println("Expected 20000.0. Response from the Client API call is " + responseBack12);
assertEquals(20000.0, responseBack12, 0.00);
}
}
/**
* Provides a set of operations on the database server
*/
interface TestE2EIntegerParaReturnDouble {
/**
* Creates a TestE2EIntegerParaReturnDouble object for executing operations on the database server.
*
* The DatabaseClientFactory class can create the DatabaseClient parameter. A single
* client object can be used for any number of requests and in multiple threads.
*
* @param db provides a client for communicating with the database server
* @return an object for session state
*/
static TestE2EIntegerParaReturnDouble on(DatabaseClient db) {
final class TestE2EIntegerParaReturnDoubleImpl implements TestE2EIntegerParaReturnDouble {
private BaseProxy baseProxy;
private TestE2EIntegerParaReturnDoubleImpl(DatabaseClient dbClient) {
baseProxy = new BaseProxy(dbClient, "/ext/TestE2EIntegerParamReturnDouble/");
}
@Override
public Double TestE2EItemPrice(Integer itemId) {
return BaseProxy.DoubleType.toDouble(
baseProxy
.request("TestE2EIntegerParamReturnDouble.sjs", BaseProxy.ParameterValuesKind.SINGLE_ATOMIC)
.withSession()
.withParams(
BaseProxy.atomicParam("itemId", true, BaseProxy.IntegerType.fromInteger(itemId)))
.withMethod("POST")
.responseSingle(true, null)
);
}
@Override
public java.lang.Float TestE2EItemPriceErrorCond(Integer items, java.lang.Integer price) {
return BaseProxy.FloatType.toFloat(
baseProxy
.request("TestE2EIntegerParamReturnDoubleErrorCond.sjs", BaseProxy.ParameterValuesKind.MULTIPLE_ATOMICS)
.withSession()
.withParams(
BaseProxy.atomicParam("items", true, BaseProxy.IntegerType.fromInteger(items)),
BaseProxy.atomicParam("price", true, BaseProxy.IntegerType.fromInteger(price)))
.withMethod("POST")
.responseSingle(true, null)
);
}
@Override
public java.lang.Float TestE2ERequiredParam(Integer items, java.lang.Integer price) {
return BaseProxy.FloatType.toFloat(
baseProxy
.request("TestRequiredParam.sjs", BaseProxy.ParameterValuesKind.MULTIPLE_ATOMICS)
.withSession()
.withParams(
BaseProxy.atomicParam("items", false, BaseProxy.IntegerType.fromInteger(items)),
BaseProxy.atomicParam("price", false, BaseProxy.IntegerType.fromInteger(price)))
.withMethod("POST")
.responseSingle(true, null)
);
}
}
return new TestE2EIntegerParaReturnDoubleImpl(db);
}
/**
* Invokes the TestE2EItemPrice operation on the database server
*
* @param itemId provides input
* @return as output
*/
Double TestE2EItemPrice(Integer itemId);
/**
* Invokes the TestE2EItemPriceErrorCond operation on the database server
*
* @param items provides input
* @param price provides input
* @return as output
*/
java.lang.Float TestE2EItemPriceErrorCond(Integer items, java.lang.Integer price);
/**
* Invokes the TestE2ERequiredParam operation on the database server
*
* @param items provides input
* @param price provides input
* @return as output
*/
java.lang.Float TestE2ERequiredParam(Integer items, java.lang.Integer price);
}
/**
* This class verifies the stream of Json documents written into DB and accepts a string query. REturns URIs
*/
interface TestE2EMultipleStringsInMultipleStringsout {
/**
* Creates a TestE2EMultipleStringsInMultipleStringsout object for executing operations on the database server.
*
* The DatabaseClientFactory class can create the DatabaseClient parameter. A single
* client object can be used for any number of requests and in multiple threads.
*
* @param db provides a client for communicating with the database server
* @return an object for session state
*/
static TestE2EMultipleStringsInMultipleStringsout on(DatabaseClient db) {
final class TestE2EMultipleStringsInMultipleStringsoutImpl implements TestE2EMultipleStringsInMultipleStringsout {
private BaseProxy baseProxy;
private TestE2EMultipleStringsInMultipleStringsoutImpl(DatabaseClient dbClient) {
baseProxy = new BaseProxy(dbClient, "/ext/TestE2EMultiStringsInStringsOut/");
}
@Override
public com.fasterxml.jackson.databind.node.ArrayNode stringsInAndStringOutAsArray(Stream<String> inputFiles, Stream<String> uris, Stream<String> searchItem) {
return BaseProxy.ArrayType.toArrayNode(
baseProxy
.request("TestE2EJsonStringsInStringsOut.sjs", BaseProxy.ParameterValuesKind.MULTIPLE_ATOMICS)
.withSession()
.withParams(
BaseProxy.atomicParam("inputFiles", false, BaseProxy.StringType.fromString(inputFiles)),
BaseProxy.atomicParam("uris", false, BaseProxy.StringType.fromString(uris)),
BaseProxy.atomicParam("searchItem", false, BaseProxy.StringType.fromString(searchItem)))
.withMethod("POST")
.responseSingle(false, Format.JSON)
);
}
}
return new TestE2EMultipleStringsInMultipleStringsoutImpl(db);
}
/**
* Invokes the stringsInAndStringOutAsArray operation on the database server
*
* @param inputFiles A sequence of strings that need to be written to Database
* @param uris A sequence of document uris for docs written to Database
* @param searchItem Search string with 1 word or multiples
* @return Module to return array of doc Ids
*/
com.fasterxml.jackson.databind.node.ArrayNode stringsInAndStringOutAsArray(Stream<String> inputFiles, Stream<String> uris, Stream<String> searchItem);
}
/**
* This class verifies the error message when module is not available
*/
interface TestE2EModuleNotFound {
/**
* Creates a TestE2EModuleNotFound object for executing operations on the database server.
*
* The DatabaseClientFactory class can create the DatabaseClient parameter. A single
* client object can be used for any number of requests and in multiple threads.
*
* @param db provides a client for communicating with the database server
* @return an object for session state
*/
static TestE2EModuleNotFound on(DatabaseClient db) {
final class TestE2EModuleNotFoundImpl implements TestE2EModuleNotFound {
private BaseProxy baseProxy;
private TestE2EModuleNotFoundImpl(DatabaseClient dbClient) {
baseProxy = new BaseProxy(dbClient, "/ext/TestE2EModuleNotFound/");
}
@Override
public java.lang.String ModuleNotFound(String count, String starttime) {
return BaseProxy.TextDocumentType.toString(
baseProxy
.request("TestE2EModuleNotFound.sjs", BaseProxy.ParameterValuesKind.MULTIPLE_ATOMICS)
.withSession()
.withParams(
BaseProxy.atomicParam("count", false, BaseProxy.DecimalType.fromString(count)),
BaseProxy.atomicParam("starttime", false, BaseProxy.TimeType.fromString(starttime)))
.withMethod("POST")
.responseSingle(false, Format.TEXT)
);
}
}
return new TestE2EModuleNotFoundImpl(db);
}
/**
* Invokes the ModuleNotFound operation on the database server
*
* @param count Big Decimal numbers
* @param starttime Java Local time
* @return Module to return text document
*/
java.lang.String ModuleNotFound(String count, String starttime);
}
/**
* This class verifies the sessions
*/
interface TestE2ESession {
/**
* Creates a TestE2ESession object for executing operations on the database server.
*
* The DatabaseClientFactory class can create the DatabaseClient parameter. A single
* client object can be used for any number of requests and in multiple threads.
*
* @param db provides a client for communicating with the database server
* @return an object for session state
*/
static TestE2ESession on(DatabaseClient db) {
final class TestE2ESessionImpl implements TestE2ESession {
private BaseProxy baseProxy;
private TestE2ESessionImpl(DatabaseClient dbClient) {
baseProxy = new BaseProxy(dbClient, "/ext/TestE2ESession/");
}
@Override
public SessionState newSessionState() {
return baseProxy.newSessionState();
}
@Override
public void SessionChecks(SessionState api_session, String uri, String content) {
baseProxy
.request("TestE2ESession.sjs", BaseProxy.ParameterValuesKind.MULTIPLE_ATOMICS)
.withSession("api_session", api_session, false)
.withParams(
BaseProxy.atomicParam("uri", false, BaseProxy.StringType.fromString(uri)),
BaseProxy.atomicParam("content", false, BaseProxy.StringType.fromString(content)))
.withMethod("POST")
.responseNone();
}
}
return new TestE2ESessionImpl(db);
}
/**
* Creates an object to track a session for a set of operations
* that require session state on the database server.
*
* @return an object for session state
*/
SessionState newSessionState();
/**
* Invokes the SessionChecks operation on the database server
*
* @param api_session Holds the session object
* @param uri Doc Id of the inserted document
* @param content Doc contents of the inserted document
*
*/
void SessionChecks(SessionState api_session, String uri, String content);
}
To test early, it would be beneficial to run the tests with both middle tier and database cluster in the cloud before DHaaS is available.
The tests should use ALB.
Similarly, we should run the NiFi tests in the DHaaS environment.