MQE-335: Headless Browser Spike

ivy00johns · ivy00johns · commit 5cbbdd37a2ec · 2017-10-06T09:35:55.000-05:00
- Updating all @env tags for tests. Only listing the ones that the test works in currently. - Updating README. Replacing "Acceptance" with "Functional". - Adding Headless command to robo.
diff --git a/dev/tests/acceptance/RoboFile.php b/dev/tests/acceptance/RoboFile.php
@@ -44,39 +44,49 @@ function generateTests()
     }
 
     /**
-     * Run all Acceptance tests using the Chrome environment
+     * Run all Functional tests using the Chrome environment
      */
     function chrome()
     {
         $this->_exec('./vendor/bin/codecept run functional --env chrome --skip-group skip');
     }
 
     /**
-     * Run all Acceptance tests using the FireFox environment
+     * Run all Functional tests using the FireFox environment
      */
     function firefox()
     {
         $this->_exec('./vendor/bin/codecept run functional --env firefox --skip-group skip');
     }
 
     /**
-     * Run all Acceptance tests using the PhantomJS environment
+     * Run all Functional tests using the PhantomJS environment
      */
     function phantomjs()
     {
         $this->_exec('./vendor/bin/codecept run functional --env phantomjs --skip-group skip');
     }
 
+    /**
+     * Run all Functional tests using the Chrome Headless environment
+     */
+    function headless()
+    {
+        $this->_exec('./vendor/bin/codecept run functional --env headless --skip-group skip');
+    }
+
     /**
      * Run all Tests with the specified @group tag, excluding @group 'skip', using the Chrome environment
+     * @param string $args
      */
     function group($args = '')
     {
         $this->taskExec('./vendor/bin/codecept run functional --verbose --steps --env chrome --skip-group skip --group')->args($args)->run();
     }
 
     /**
-     * Run all Acceptance tests located under the Directory Path provided using the Chrome environment
+     * Run all Functional tests located under the Directory Path provided using the Chrome environment
+     * @param string $args
      */
     function folder($args = '')
     {
diff --git a/dev/tests/acceptance/tests/functional.suite.dist.yml b/dev/tests/acceptance/tests/functional.suite.dist.yml
@@ -26,7 +26,7 @@ modules:
         \Magento\FunctionalTestingFramework\Module\MagentoWebDriver:
             url: "%MAGENTO_BASE_URL%"
             backend_name: admin
-            browser: chrome
+            browser: 'chrome'
             window_size: maximize
             username: "%MAGENTO_ADMIN_USERNAME%"
             password: "%MAGENTO_ADMIN_PASSWORD%"
diff --git a/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/Backend/Cest/AdminLoginCest.xml b/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/Backend/Cest/AdminLoginCest.xml
@@ -12,18 +12,19 @@
         <annotations>
             <features value="Admin Login"/>
             <stories value="Login on the Admin Login page"/>
-            <group value="example"/>
-            <group value="login"/>
-            <env value="chrome"/>
-            <env value="firefox"/>
-            <env value="phantomjs"/>
         </annotations>
         <test name="LoginAsAdmin">
             <annotations>
                 <title value="You should be able to log into the Magento Admin backend."/>
                 <description value="You should be able to log into the Magento Admin backend."/>
                 <severity value="CRITICAL"/>
                 <testCaseId value="MAGETWO-71572"/>
+                <group value="example"/>
+                <group value="login"/>
+                <env value="chrome"/>
+                <env value="firefox"/>
+                <env value="phantomjs"/>
+                <env value="headless"/>
             </annotations>
             <amOnPage url="{{AdminLoginPage}}" mergeKey="amOnAdminLoginPage"/>
             <fillField selector="{{AdminLoginFormSection.username}}" userInput="{{_ENV.MAGENTO_ADMIN_USERNAME}}" mergeKey="fillUsername"/>
diff --git a/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/Catalog/Cest/AdminCreateCategoryCest.xml b/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/Catalog/Cest/AdminCreateCategoryCest.xml
@@ -12,10 +12,6 @@
         <annotations>
             <features value="Category Creation"/>
             <stories value="Create a Category via the Admin"/>
-            <group value="category"/>
-            <env value="chrome"/>
-            <env value="firefox"/>
-            <env value="phantomjs"/>
         </annotations>
         <after>
             <amOnPage url="admin/admin/auth/logout/" mergeKey="amOnLogoutPage"/>
@@ -26,6 +22,9 @@
                 <description value="You should be able to create a Category in the admin back-end."/>
                 <severity value="CRITICAL"/>
                 <testCaseId value="MAGETWO-72102"/>
+                <group value="category"/>
+                <env value="chrome"/>
+                <env value="headless"/>
             </annotations>
             <amOnPage url="{{AdminLoginPage}}" mergeKey="amOnAdminLoginPage"/>
             <fillField selector="{{AdminLoginFormSection.username}}" userInput="{{_ENV.MAGENTO_ADMIN_USERNAME}}" mergeKey="fillUsername"/>
diff --git a/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/Catalog/Cest/AdminCreateConfigurableProductCest.xml b/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/Catalog/Cest/AdminCreateConfigurableProductCest.xml
@@ -12,11 +12,6 @@
         <annotations>
             <features value="Product Creation"/>
             <stories value="Create a Configurable Product via the Admin"/>
-            <group value="configurable"/>
-            <group value="product"/>
-            <env value="chrome"/>
-            <env value="firefox"/>
-            <env value="phantomjs"/>
         </annotations>
         <before>
             <loginAsAdmin mergeKey="loginAsAdmin"/>
@@ -30,6 +25,9 @@
                 <description value="You should be able to create a Configurable Product via the Admin."/>
                 <severity value="CRITICAL"/>
                 <testCaseId value="MAGETWO-26041"/>
+                <group value="configurable"/>
+                <group value="product"/>
+                <env value="chrome"/>
             </annotations>
 
             <amOnPage url="{{AdminCategoryPage.urlPath}}" mergeKey="amOnCategoryGridPage"/>
diff --git a/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/Catalog/Cest/AdminCreateSimpleProductCest.xml b/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/Catalog/Cest/AdminCreateSimpleProductCest.xml
@@ -12,10 +12,6 @@
         <annotations>
             <features value="Product Creation"/>
             <stories value="Create a Simple Product via Admin"/>
-            <group value="product"/>
-            <env value="chrome"/>
-            <env value="firefox"/>
-            <env value="phantomjs"/>
         </annotations>
         <before>
             <createData entity="_defaultCategory" mergeKey="createPreReqCategory"/>
@@ -26,6 +22,9 @@
                 <description value="You should be able to create a Simple Product in the admin back-end."/>
                 <severity value="CRITICAL"/>
                 <testCaseId value="MAGETWO-23414"/>
+                <group value="product"/>
+                <env value="chrome"/>
+                <env value="headless"/>
             </annotations>
             <amOnPage url="{{AdminLoginPage.url}}" mergeKey="navigateToAdmin"/>
             <fillField userInput="{{_ENV.MAGENTO_ADMIN_USERNAME}}" selector="{{AdminLoginFormSection.username}}" mergeKey="fillUsername"/>
diff --git a/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/Checkout/Cest/StorefrontCustomerCheckoutCest.xml b/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/Checkout/Cest/StorefrontCustomerCheckoutCest.xml
@@ -12,10 +12,6 @@
         <annotations>
             <features value="Checkout"/>
             <stories value="Checkout via the Admin"/>
-            <group value="checkout"/>
-            <env value="chrome"/>
-            <env value="firefox"/>
-            <env value="phantomjs"/>
         </annotations>
         <before>
             <createData entity="SimpleSubCategory" mergeKey="simplecategory"/>
@@ -35,6 +31,9 @@
                 <description value="Should be able to place an order as a customer."/>
                 <severity value="CRITICAL"/>
                 <testCaseId value="#"/>
+                <group value="checkout"/>
+                <env value="chrome"/>
+                <env value="headless"/>
             </annotations>
 
             <amOnPage mergeKey="s1"  url="customer/account/login/"/>
diff --git a/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/Checkout/Cest/StorefrontGuestCheckoutCest.xml b/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/Checkout/Cest/StorefrontGuestCheckoutCest.xml
@@ -12,10 +12,6 @@
         <annotations>
             <features value="Checkout"/>
             <stories value="Checkout via Guest Checkout"/>
-            <group value="checkout"/>
-            <env value="chrome"/>
-            <env value="firefox"/>
-            <env value="phantomjs"/>
         </annotations>
         <before>
             <createData entity="_defaultCategory" mergeKey="createCategory"/>
@@ -33,6 +29,9 @@
                 <description value="Should be able to place an order as a Guest."/>
                 <severity value="CRITICAL"/>
                 <testCaseId value="MAGETWO-72094"/>
+                <group value="checkout"/>
+                <env value="chrome"/>
+                <env value="headless"/>
             </annotations>
             <amOnPage url="{{StorefrontCategoryPage.url($$createCategory.name$$)}}" mergeKey="onCategoryPage"/>
             <waitForPageLoad mergeKey="waitForPageLoad1"/>
diff --git a/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/Cms/Cest/AdminCreateCmsPageCest.xml b/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/Cms/Cest/AdminCreateCmsPageCest.xml
@@ -12,10 +12,6 @@
         <annotations>
             <features value="CMS Page Creation"/>
             <stories value="Create a CMS Page via the Admin"/>
-            <group value="cms"/>
-            <env value="chrome"/>
-            <env value="firefox"/>
-            <env value="phantomjs"/>
         </annotations>
         <after>
             <amOnPage url="admin/admin/auth/logout/" mergeKey="amOnLogoutPage"/>
@@ -26,6 +22,10 @@
                 <description value="You should be able to create a CMS Page via the Admin."/>
                 <severity value="CRITICAL"/>
                 <testCaseId value="MAGETWO-25580"/>
+                <group value="cms"/>
+                <env value="chrome"/>
+                <env value="firefox"/>
+                <env value="headless"/>
             </annotations>
             <amOnPage url="{{AdminLoginPage}}" mergeKey="navigateToAdmin"/>
             <fillField selector="{{AdminLoginFormSection.username}}" userInput="{{_ENV.MAGENTO_ADMIN_USERNAME}}" mergeKey="fillUsername"/>
diff --git a/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/Customer/Cest/AdminCreateCustomerCest.xml b/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/Customer/Cest/AdminCreateCustomerCest.xml
@@ -12,17 +12,17 @@
         <annotations>
             <features value="Customer Creation"/>
             <stories value="Create a Customer via the Admin"/>
-            <group value="customer"/>
-            <env value="chrome"/>
-            <env value="firefox"/>
-            <env value="phantomjs"/>
         </annotations>
         <test name="CreateCustomerViaAdminCest">
             <annotations>
                 <title value="You should be able to create a customer in the Admin back-end."/>
                 <description value="You should be able to create a customer in the Admin back-end."/>
                 <severity value="CRITICAL"/>
                 <testCaseId value="MAGETWO-72095"/>
+                <group value="customer"/>
+                <group value="create"/>
+                <env value="chrome"/>
+                <env value="headless"/>
             </annotations>
             <amOnPage url="{{AdminLoginPage.url}}" mergeKey="navigateToAdmin"/>
             <fillField userInput="{{_ENV.MAGENTO_ADMIN_USERNAME}}" selector="{{AdminLoginFormSection.username}}" mergeKey="fillUsername"/>
diff --git a/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/Customer/Cest/StorefrontCreateCustomerCest.xml b/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/Customer/Cest/StorefrontCreateCustomerCest.xml
@@ -12,17 +12,18 @@
         <annotations>
             <features value="Customer Creation"/>
             <stories value="Create a Customer via the Storefront"/>
-            <env value="chrome"/>
-            <env value="firefox"/>
-            <env value="phantomjs"/>
         </annotations>
         <test name="CreateCustomerViaStorefront">
             <annotations>
                 <title value="You should be able to create a customer via the storefront"/>
                 <description value="You should be able to create a customer via the storefront."/>
                 <severity value="CRITICAL"/>
-                <group value="create"/>
                 <testCaseId value="MAGETWO-23546"/>
+                <group value="customer"/>
+                <group value="create"/>
+                <env value="chrome"/>
+                <env value="firefox"/>
+                <env value="headless"/>
             </annotations>
             <amOnPage mergeKey="amOnStorefrontPage"  url="/"/>
             <click mergeKey="clickOnCreateAccountLink" selector="{{StorefrontPanelHeaderSection.createAnAccountLink}}"/>
diff --git a/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/Customer/Cest/StorefrontExistingCustomerLoginCest.xml b/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/Customer/Cest/StorefrontExistingCustomerLoginCest.xml
@@ -12,10 +12,6 @@
         <annotations>
             <features value="Customer Login"/>
             <stories value="Existing Customer can Login on the Storefront"/>
-            <group value="customer"/>
-            <env value="chrome"/>
-            <env value="firefox"/>
-            <env value="phantomjs"/>
         </annotations>
         <before>
             <createData entity="Simple_US_Customer" mergeKey="Simple_US_Customer"/>
@@ -29,6 +25,11 @@
                 <description value="You should be able to create a customer via the storefront."/>
                 <severity value="CRITICAL"/>
                 <testCaseId value="MAGETWO-72103"/>
+                <group value="customer"/>
+                <env value="chrome"/>
+                <env value="firefox"/>
+                <env value="phantomjs"/>
+                <env value="headless"/>
             </annotations>
             <amOnPage mergeKey="amOnSignInPage"  url="customer/account/login/"/>
             <fillField  mergeKey="fillEmail" userInput="$$Simple_US_Customer.email$$" selector="{{StorefrontCustomerSignInFormSection.emailField}}"/>
diff --git a/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/Sales/Cest/AdminCreateInvoiceCest.xml b/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/Sales/Cest/AdminCreateInvoiceCest.xml
@@ -12,10 +12,6 @@
         <annotations>
             <features value="Invoice Creation"/>
             <stories value="Create an Invoice via the Admin"/>
-            <group value="sales"/>
-            <env value="chrome"/>
-            <env value="firefox"/>
-            <env value="phantomjs"/>
         </annotations>
         <before>
             <createData entity="_defaultCategory" mergeKey="createCategory"/>
@@ -30,6 +26,9 @@
                 <description value="Should be able to create an invoice via the admin."/>
                 <severity value="NORMAL"/>
                 <testCaseId value="MAGETWO-72096"/>
+                <group value="sales"/>
+                <env value="chrome"/>
+                <env value="headless"/>
             </annotations>
 
             <!-- todo: Create an order via the api instead of driving the browser  -->
diff --git a/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/SampleTemplates/Cest/TemplateCestFile.xml b/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/SampleTemplates/Cest/TemplateCestFile.xml
@@ -12,8 +12,6 @@
         <annotations>
             <features value=""/>
             <stories value=""/>
-            <group value=""/>
-            <env value=""/>
         </annotations>
         <before>
 
@@ -27,6 +25,8 @@
                 <description value=""/>
                 <severity value=""/>
                 <testCaseId value=""/>
+                <group value=""/>
+                <env value=""/>
             </annotations>
             <!-- ADD TEST STEPS HERE -->
         </test>
diff --git a/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/SampleTests/Cest/MinimumTestCest.xml b/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/SampleTests/Cest/MinimumTestCest.xml
@@ -12,10 +12,6 @@
         <annotations>
             <features value="Minimum Test"/>
             <stories value="Minimum Test"/>
-            <group value="example"/>
-            <env value="chrome"/>
-            <env value="firefox"/>
-            <env value="phantomjs"/>
         </annotations>
         <after>
             <seeInCurrentUrl url="/admin/admin/" mergeKey="seeInCurrentUrl"/>
@@ -24,6 +20,11 @@
             <annotations>
                 <title value="Minimum Test"/>
                 <description value="Minimum Test"/>
+                <group value="example"/>
+                <env value="chrome"/>
+                <env value="firefox"/>
+                <env value="phantomjs"/>
+                <env value="headless"/>
             </annotations>
             <amOnPage url="{{AdminLoginPage}}" mergeKey="navigateToAdmin"/>
             <fillField selector="{{AdminLoginFormSection.username}}" userInput="{{_ENV.MAGENTO_ADMIN_USERNAME}}" mergeKey="fillUsername"/>
diff --git a/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/SampleTests/Cest/PersistMultipleEntitiesCest.xml b/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/SampleTests/Cest/PersistMultipleEntitiesCest.xml
@@ -9,10 +9,6 @@
 <config xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:noNamespaceSchemaLocation="../../../../../../vendor/magento/magento2-functional-testing-framework/src/Magento/FunctionalTestingFramework/Test/etc/testSchema.xsd">
     <cest name="PersistMultipleEntitiesCest">
-        <annotations>
-            <group value="ff"/>
-            <group value="skip"/>
-        </annotations>
         <before>
             <createData entity="simplesubcategory" mergeKey="simplecategory"/>
             <createData entity="simpleproduct" mergeKey="simpleproduct1">
@@ -28,6 +24,10 @@
             <deleteData createDataKey="simplecategory" mergeKey="deleteCategory"/>
         </after>
         <test name="PersistMultipleEntitiesTest">
+            <annotations>
+                <group value="ff"/>
+                <group value="skip"/>
+            </annotations>
             <amOnPage mergeKey="s11" url="/$$simplecategory.name$$.html" />
             <waitForPageLoad mergeKey="s33"/>
             <see mergeKey="s35" selector="{{StorefrontCategoryMainSection.productCount}}" userInput="2"/>

Original file line number	Diff line number	Diff line change
`@@ -44,39 +44,49 @@ function generateTests()`
`44`	`44`	`}`
`45`	`45`
`46`	`46`	`/**`
`47`		`- * Run all Acceptance tests using the Chrome environment`
	`47`	`+ * Run all Functional tests using the Chrome environment`
`48`	`48`	`*/`
`49`	`49`	`function chrome()`
`50`	`50`	`{`
`51`	`51`	`$this->_exec('./vendor/bin/codecept run functional --env chrome --skip-group skip');`
`52`	`52`	`}`
`53`	`53`
`54`	`54`	`/**`
`55`		`- * Run all Acceptance tests using the FireFox environment`
	`55`	`+ * Run all Functional tests using the FireFox environment`
`56`	`56`	`*/`
`57`	`57`	`function firefox()`
`58`	`58`	`{`
`59`	`59`	`$this->_exec('./vendor/bin/codecept run functional --env firefox --skip-group skip');`
`60`	`60`	`}`
`61`	`61`
`62`	`62`	`/**`
`63`		`- * Run all Acceptance tests using the PhantomJS environment`
	`63`	`+ * Run all Functional tests using the PhantomJS environment`
`64`	`64`	`*/`
`65`	`65`	`function phantomjs()`
`66`	`66`	`{`
`67`	`67`	`$this->_exec('./vendor/bin/codecept run functional --env phantomjs --skip-group skip');`
`68`	`68`	`}`
`69`	`69`
	`70`	`+ /**`
	`71`	`+ * Run all Functional tests using the Chrome Headless environment`
	`72`	`+ */`
	`73`	`+ function headless()`
	`74`	`+ {`
	`75`	`+ $this->_exec('./vendor/bin/codecept run functional --env headless --skip-group skip');`
	`76`	`+ }`
	`77`	`+`
`70`	`78`	`/**`
`71`	`79`	`* Run all Tests with the specified @group tag, excluding @group 'skip', using the Chrome environment`
	`80`	`+ * @param string $args`
`72`	`81`	`*/`
`73`	`82`	`function group($args = '')`
`74`	`83`	`{`
`75`	`84`	`$this->taskExec('./vendor/bin/codecept run functional --verbose --steps --env chrome --skip-group skip --group')->args($args)->run();`
`76`	`85`	`}`
`77`	`86`
`78`	`87`	`/**`
`79`		`- * Run all Acceptance tests located under the Directory Path provided using the Chrome environment`
	`88`	`+ * Run all Functional tests located under the Directory Path provided using the Chrome environment`
	`89`	`+ * @param string $args`
`80`	`90`	`*/`
`81`	`91`	`function folder($args = '')`
`82`	`92`	`{`