Multi-dimensonal testing

[shhsu]

The cold start tests has too many files now

https://github.com/aspnet/Wave/tree/shhsu/coldstart/client/test/coldstart

It would be a nightmare to maintain I propose we only keep the md file along with a file like this

run-set-coldstart.json

{
    "defaults" :
    {
            "scriptSource": "https://github.com/aspnet/Performance.git",
            "testAppBranch" : "shhsu/coldstart",
            "testAppSource" : "https://github.com/aspnet/Performance.git",
            "testAppDir" : "null"
    },

    "dimension" :
    [
        {
            "name" : "os",
            "attributes" :
            [
                {
                    "name" : "linux",
                    "feature" : {
                        "logdir" : "/home/asplab/$(testid)",
                        "path" : "/home/asplab/.dotnet",
                        "rebootCommand" : "sudo shutdown -r now",
                        "measureScript" : "./Measure.sh",
                        "publishScript" : "./Publish.sh",
                        "archiveScript" : "./Archive.sh",

                        "gitHome" : "/home/asplab/git/aspnet",
                        "perfHome" : "/home/asplab/git/aspnet/Performance",
                        "scriptHome" : "/home/asplab/git/aspnet/Performance/test/ColdStart",
                        "testAppHome" : "/home/asplab/git/aspnet/Performance",

                        "$targets" : {
                            "server" : {
                                "name" : "Asp-xHPSLC1"
                            }
                        }
                    }
                },
                {
                    "name" : "windows",
                    "feature" : {
                        "logdir": "c:\\$(testid)",
                        "path" : "%localappdata%\\microsoft\\dotnet;%ProgramFiles%\\Git\\cmd",

                        "rebootCommand" : "shutdown /r /f /t 0",
                        "measureScript" : "Measure.ps1",
                        "publishScript" : "Publish.ps1",
                        "archiveScript" : "Archive.ps1",

                        "gitHome" : "D:\\git\\aspnet",
                        "perfHome" : "D:\\git\\aspnet\\Performance",
                        "scriptHome" : "D:\\git\\aspnet\\Performance\\test\\ColdStart",
                        "testAppHome" : "D:\\git\\aspnet\\Performance"

                        "$targets": {
                            "server": {
                                "name": "Asp-xHPc7"
                            }
                        }
                    }
                }
            ]
        },
        {
            "name" : "host",
            "attributes" :
            [
                { "name" : "self" },
                {
                    "name" : "iis",
                    "pivot" :
                    [   
                        {
                            "on" : { "os" : { "in" : [ "windows" ] } },
                            "feature" :
                            {
                                "measureScript" : "Measure-IIS.ps1",
                                "publishScript" : "Publish-IIS.ps1",
                                "archiveScript" : "Archive-IIS.ps1"
                            }
                        }
                    ],
                },
                {
                    "name" : "nginx",
                    "pivot" :
                    [
                        {
                            "on" : {"os" : { "in" : [ "linux" ] } },
                            "feature" :
                            {
                                "measureScript" : "./Measure-nginx.sh",
                                "archiveScript" : "./Archive-nginx.sh"
                            }
                        }
                    ]
                },
            ]
        },
        {
            "name" : "app",
            "attributes" :
            [
                {
                    "name" : "text",
                    "feature" : { "targetApp" : "BasicKestrel" }
                },
                {
                    "name" : "mvc",
                    "feature" : { "targetApp" : "HelloWorldMvc" }
                },
                {
                    "name" : "musicstore",
                    "feature" : { "targetApp" : "MusicStore" },
                    "pivot" :
                    [
                        {
                            "on" : { "os" : { "eq" : "windows" } },
                            "feature" :
                            {
                                "testAppHome" : "D:\\git\\aspnet\\MusicStore",
                                "testAppDir" : "D:\\git\\aspnet\\MusicStore\\src\\MusicStore"
                            }
                        },
                        {
                            "on" : { "os" : { "eq", "linux" } },
                            "feature" :
                            {
                                "testAppHome" : "/home/asplab/git/aspnet/MusicStore",
                                "testAppDir" : "/home/asplab/git/aspnet/MusicStore/src/MusicStore"
                            }
                        }
                    ]
                }
            ]
        },
        {
            "name" : "framework",
            "attributes" :
            [
                {
                    "name" : "core",
                    "feature" : { "framework" : "netcoreapp1.0" }
                },
                {
                    "name" : "desktop",
                    "pivot" :
                    {
                        "on" : { "os" : { "eq", "windows" } },
                        "feature" : { "framework" : "net451" }
                    }
                }
            ]
        },
        {
            "name" : "db",
            "attributes" :
            [
                {
                    "name" : "remotedb",
                    "pivot" :
                    {
                        "on" : { "targetApp" : { "eq" : "MusicStore" } },
                        "feature" :
                        {
                            "testAppBranch" : "shhsu/coldstart_remote_sql2016"
                        }
                    }
                },
                {
                    "name" : "localdb",
                    "pivot" :
                    {
                        "on" : { { "targetApp" : { "eq" : "MusicStore"} }, { "os" : { "eq" : "Windows" } } },
                        "feature" : {}                        
                    }
                },
                {
                    "name" : "inmemory",
                    "pivot" :
                    {
                        "on" : { "targetApp" : { "eq" : "MusicStore" } },
                        "feature" :
                        {
                            "testAppBranch" : "shhsu/coldstart_inmemory"
                        }
                    }
                },
                {
                    "name" : "nodb", 
                    "pivot" :
                    {
                        "on" : { "targetApp" : { "ne", "MusicStore" } },
                        "feature" : {}                        
                    }
                }
            ]
        }
    ]
}

[shhsu]

This is the file that is supposed to generate all tests contained in the coldstart folder. The aim is to use scenario generation to avoid code duplicates