From 35b38830456d1028c7bf33471fc584d0cbcd3eec Mon Sep 17 00:00:00 2001
From: Elizabeth Salita <salita_elizabeth@bah.com>
Date: Fri, 5 Mar 2021 15:26:49 -0500
Subject: [PATCH 1/7] update to COVID Data Sources & Methodology copy

---
 .../components/covid19/DataSourcesAndMethodologiesPage.jsx   | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/js/components/covid19/DataSourcesAndMethodologiesPage.jsx b/src/js/components/covid19/DataSourcesAndMethodologiesPage.jsx
index b007ee1df7..1171882bb6 100644
--- a/src/js/components/covid19/DataSourcesAndMethodologiesPage.jsx
+++ b/src/js/components/covid19/DataSourcesAndMethodologiesPage.jsx
@@ -691,7 +691,10 @@ export default () => {
                                             For <strong>Award Spending (including Loan Spending)</strong>, sum together:
                                             <ul>
                                                 <li>
-                                                    Either Gross Outlay Amount (for linked and unlinked data) or Outlayed Amount Funded by COVID-19 Supplementals (for linked data only) for every award ID (award unique key) tagged with a COVID-19 DEFC.
+                                                    Outlayed Amount Funded by COVID-19 Supplementals for every award ID (award unique key) tagged with a COVID-19 DEFC [for Linked Awards only].
+                                                </li>
+                                                <li>
+                                                    Gross Outlay Amount, Downward Adjustments of Prior Year Prepaid Advanced Undelivered Orders and Obligation Refunds Collected, and Downward Adjustments of Prior Year Paid Delivered Orders and Obligations Refunds Collected for every award ID (award unique key) tagged with a COVID-19 DEFC [for Linked and Unlinked Awards].
                                                 </li>
                                                 <li>
                                                     Filter by any award type as desired.

From 4f485f6d69908a6d69dbba2c1cbdb9f2deb4006a Mon Sep 17 00:00:00 2001
From: Maxwell Kendall <maxwell.n.kendall@gmail.com>
Date: Fri, 19 Feb 2021 13:29:16 -0500
Subject: [PATCH 2/7] [DEV-6937] Implementing canonical tag & adding tests for
 helper fns.

---
 .../sharedComponents/metaTags/MetaTags.jsx    | 20 +++++++++++--------
 src/js/helpers/metaTagHelper.js               | 13 ++++++++++++
 tests/helpers/metaTagHelper-test.js           | 19 ++++++++++++++++++
 3 files changed, 44 insertions(+), 8 deletions(-)
 create mode 100644 tests/helpers/metaTagHelper-test.js

diff --git a/src/js/components/sharedComponents/metaTags/MetaTags.jsx b/src/js/components/sharedComponents/metaTags/MetaTags.jsx
index 50cb2707de..eb6da45886 100644
--- a/src/js/components/sharedComponents/metaTags/MetaTags.jsx
+++ b/src/js/components/sharedComponents/metaTags/MetaTags.jsx
@@ -9,6 +9,10 @@ import { useLocation } from 'react-router-dom';
 
 import Analytics from 'helpers/analytics/Analytics';
 import { Helmet } from 'react-helmet';
+import {
+    isCustomPageTitleDefined,
+    getCanonicalUrl
+} from 'helpers/metaTagHelper';
 
 const propTypes = {
     og_url: PropTypes.string,
@@ -19,19 +23,13 @@ const propTypes = {
 };
 
 const defaultProps = {
-    og_url: 'https://usaspending.gov/',
+    og_url: 'https://usaspending.gov',
     og_title: 'USAspending.gov',
     og_description: 'USAspending.gov is the new official source of accessible, searchable and reliable spending data for the U.S. Government.',
     og_site_name: 'USAspending.gov',
     og_image: 'https://usaspending.gov/img/FacebookOG.png'
 };
 
-const isCustomPageTitleDefined = (title = "USAspending.gov") => {
-    if (title === "USAspending.gov") return false;
-    if (title.split('|')[0] === ' ') return false;
-    return true;
-};
-
 const MetaTags = ({
     og_url: url,
     og_title: title,
@@ -40,6 +38,7 @@ const MetaTags = ({
     og_image: image
 }) => {
     const { pathname } = useLocation();
+
     const [tags, setTags] = useState([]);
 
     const generateTags = () => {
@@ -78,7 +77,12 @@ const MetaTags = ({
                 key="og_image" />);
         }
 
-        setTags(newTags);
+        setTags(
+            newTags
+                .concat([
+                    <link rel="canonical" href={getCanonicalUrl(pathname)} />
+                ])
+        );
     };
 
     useEffect(() => {
diff --git a/src/js/helpers/metaTagHelper.js b/src/js/helpers/metaTagHelper.js
index 37a099878e..37a67b772a 100644
--- a/src/js/helpers/metaTagHelper.js
+++ b/src/js/helpers/metaTagHelper.js
@@ -266,3 +266,16 @@ export const stateLandingPageMetaTags = {
 };
 
 /* eslint-enable max-len */
+
+export const isCustomPageTitleDefined = (title = "USAspending.gov") => {
+    if (title === "USAspending.gov") return false;
+    if (title.split('|')[0] === ' ') return false;
+    if (!title) return false;
+    return true;
+};
+
+const canonicalUrl = 'https://www.usaspending.gov';
+
+const trailingSlashRegExp = new RegExp(/\/$/);
+
+export const getCanonicalUrl = (path) => `${canonicalUrl}${path.replace(trailingSlashRegExp, '')}`;
diff --git a/tests/helpers/metaTagHelper-test.js b/tests/helpers/metaTagHelper-test.js
new file mode 100644
index 0000000000..d403873aa2
--- /dev/null
+++ b/tests/helpers/metaTagHelper-test.js
@@ -0,0 +1,19 @@
+import { isCustomPageTitleDefined, getCanonicalUrl } from 'helpers/metaTagHelper';
+
+test.each([
+    ['Test Page | USAspending.gov', true],
+    ['', false],
+    ['USAspending.gov', false],
+    [' | USAspending.gov', false],
+    ['T | USAspending.gov', true]
+])('isCustomPageTitleDefined: when input is %s, return is %s', (input, rtrn) => {
+    expect(isCustomPageTitleDefined(input)).toEqual(rtrn);
+});
+
+test.each([
+    ['/', 'https://www.usaspending.gov'],
+    ['/award/123/', 'https://www.usaspending.gov/award/123'],
+    ['/test', 'https://www.usaspending.gov/test']
+])('getCanonicalUrl: when input is %s, return is %s', (input, rtrn) => {
+    expect(getCanonicalUrl(input)).toEqual(rtrn);
+});

From 5d0af7cc431f8550ae16f657da87602c638acc30 Mon Sep 17 00:00:00 2001
From: Maxwell Kendall <maxwell.n.kendall@gmail.com>
Date: Fri, 26 Feb 2021 13:12:00 -0500
Subject: [PATCH 3/7] [DEV-6937] robots.txt now excludes:

- glossary query param
- .php routes
---
 scripts/sitemaps/buildSitemap.js | 24 +++++++++++++-----------
 scripts/sitemaps/pages.js        |  2 +-
 scripts/sitemaps/robots.txt      |  0
 3 files changed, 14 insertions(+), 12 deletions(-)
 create mode 100644 scripts/sitemaps/robots.txt

diff --git a/scripts/sitemaps/buildSitemap.js b/scripts/sitemaps/buildSitemap.js
index b4fd408175..000c0a5726 100644
--- a/scripts/sitemaps/buildSitemap.js
+++ b/scripts/sitemaps/buildSitemap.js
@@ -66,7 +66,7 @@ const createSitemapEntry = (xml, pageData, pageInfo) => {
 const createRobots = () => {
     fs.writeFile(
         path.resolve(__dirname, `./sitemapFiles/robots.txt`),
-        `User-agent: * \nAllow: /\n\nSitemap: ${siteUrl}/sitemap.xml`,
+        `User-agent: * \nDisallow: /*.php*\nDisallow: /*?glossary=*\n\nSitemap: ${siteUrl}/sitemap.xml`,
         (e) => {
             if (e) {
                 console.log(' Error : ', e);
@@ -219,13 +219,15 @@ const createSitemapDirectory = () => new Promise(
         })
 );
 
-createSitemapDirectory()
-    .then(() => buildIndividualSitemaps())
-    .then((individalSiteMaps) => {
-        buildIndexedSitemap(individalSiteMaps);
-        createRobots();
-    })
-    .catch((e) => {
-        console.log(`error build site maps: ${e}`);
-        throw e.message;
-    });
+createRobots();
+
+// createSitemapDirectory()
+//     .then(() => buildIndividualSitemaps())
+//     .then((individalSiteMaps) => {
+//         buildIndexedSitemap(individalSiteMaps);
+//         createRobots();
+//     })
+//     .catch((e) => {
+//         console.log(`error build site maps: ${e}`);
+//         throw e.message;
+//     });
diff --git a/scripts/sitemaps/pages.js b/scripts/sitemaps/pages.js
index a1f8ee91c3..9d0738ac33 100644
--- a/scripts/sitemaps/pages.js
+++ b/scripts/sitemaps/pages.js
@@ -1,7 +1,7 @@
 const moment = require('moment');
 
 const apiUrl = process.env.API_URL || 'https://api.usaspending.gov:443';
-const siteUrl = process.env.SITE_URL || 'https://www.usaspending.gov';
+const siteUrl = 'https://sandbox.usaspending.gov';
 
 const routes = [
     // no longer importing these from RouterRoutes because the many import statements in that file completely mess up babel
diff --git a/scripts/sitemaps/robots.txt b/scripts/sitemaps/robots.txt
new file mode 100644
index 0000000000..e69de29bb2

From 053fda0b10fd0468019ab70b810d592d10db4555 Mon Sep 17 00:00:00 2001
From: Maxwell Kendall <maxwell.n.kendall@gmail.com>
Date: Mon, 1 Mar 2021 14:13:20 -0500
Subject: [PATCH 4/7] [DEV-6937] Fixing broken links to /recipient

We were seeing duplicate URLs in the crawl for recipient/:id/latest
and recipient/id. This was just because we had some links that didn't
append /latest to the URL.

This is a simple find and replace change.
---
 .../components/award/shared/overview/Recipient.jsx   |  4 ++--
 .../recipient/RecipientMultiParentCollapse.jsx       |  6 +++---
 .../recipient/SpendingByRecipientContainer.jsx       |  6 +++---
 .../rank/RankVisualizationWrapperContainer.jsx       |  2 +-
 .../covid19/SpendingByRecipientContainer-test.jsx    | 12 ++++++------
 5 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/src/js/components/award/shared/overview/Recipient.jsx b/src/js/components/award/shared/overview/Recipient.jsx
index 051a6744a0..4c4d5347f0 100644
--- a/src/js/components/award/shared/overview/Recipient.jsx
+++ b/src/js/components/award/shared/overview/Recipient.jsx
@@ -38,10 +38,10 @@ const Recipient = ({
 
     const formatRecipientLink = (internalId, name) => {
         if (internalId && name) {
-            return (<Link to={`/recipient/${internalId}`}>{name}</Link>);
+            return (<Link to={`/recipient/${internalId}/latest`}>{name}</Link>);
         }
         else if (internalId) {
-            return (<Link to={`/recipient/${internalId}`}>Unknown</Link>);
+            return (<Link to={`/recipient/${internalId}/latest`}>Unknown</Link>);
         }
         return name;
     };
diff --git a/src/js/components/recipient/RecipientMultiParentCollapse.jsx b/src/js/components/recipient/RecipientMultiParentCollapse.jsx
index 0eef17cad7..6c26f2fed4 100644
--- a/src/js/components/recipient/RecipientMultiParentCollapse.jsx
+++ b/src/js/components/recipient/RecipientMultiParentCollapse.jsx
@@ -37,7 +37,7 @@ export default class RecipientMultiParentCollapse extends React.Component {
                 <Link
                     className="recipient-overview__multiparents"
                     key={currentValue.parent_duns}
-                    to={`/recipient/${currentValue.parent_id}`}>
+                    to={`/recipient/${currentValue.parent_id}/latest`}>
                     {currentValue.parent_name}
                     {currentValue.parent_duns ? `(${currentValue.parent_duns})` : ''}
                 </Link>
@@ -62,7 +62,7 @@ export default class RecipientMultiParentCollapse extends React.Component {
                             This recipient is a child of &nbsp;
                             <Link
                                 className="recipient-overview__parent-link"
-                                to={`/recipient/${initialParent.parent_id}`}>
+                                to={`/recipient/${initialParent.parent_id}/latest`}>
                                 {initialParent.parent_name} {initialDuns}
                             </Link>
                         </div>
@@ -104,7 +104,7 @@ export default class RecipientMultiParentCollapse extends React.Component {
                             <Link
                                 key={initialDuns}
                                 className="recipient-overview__multiparents"
-                                to={`/recipient/${initialParent.parent_id}`}>
+                                to={`/recipient/${initialParent.parent_id}/latest`}>
                                 {initialParent.parent_name} {initialDuns}
                             </Link>
                             <div className={this.state.open ? '' : 'hide'}>
diff --git a/src/js/containers/covid19/recipient/SpendingByRecipientContainer.jsx b/src/js/containers/covid19/recipient/SpendingByRecipientContainer.jsx
index 2adb2c5b37..0936fad95b 100644
--- a/src/js/containers/covid19/recipient/SpendingByRecipientContainer.jsx
+++ b/src/js/containers/covid19/recipient/SpendingByRecipientContainer.jsx
@@ -127,10 +127,10 @@ export const parseRows = (rows, activeTab, query) => (
             link = (
                 <>
                     {description}&nbsp;(
-                    <Link onClick={handleClick} to={`/recipient/${rowData._childId}`}>
+                    <Link onClick={handleClick} to={`/recipient/${rowData._childId}/latest`}>
                         as Child
                     </Link>,&nbsp;
-                    <Link onClick={handleClick} to={`/recipient/${rowData._recipientId}`}>
+                    <Link onClick={handleClick} to={`/recipient/${rowData._recipientId}/latest`}>
                         as Recipient
                     </Link>
                     )
@@ -141,7 +141,7 @@ export const parseRows = (rows, activeTab, query) => (
             const handleClick = clickedRecipientProfile(`${description}`);
             // there is a single profile page for this recipient
             link = (
-                <Link onClick={handleClick} to={`/recipient/${rowData._childId || rowData._recipientId}`}>
+                <Link onClick={handleClick} to={`/recipient/${rowData._childId || rowData._recipientId}/latest`}>
                     {description}
                 </Link>
             );
diff --git a/src/js/containers/search/visualizations/rank/RankVisualizationWrapperContainer.jsx b/src/js/containers/search/visualizations/rank/RankVisualizationWrapperContainer.jsx
index fe211510ae..4abc49e58b 100644
--- a/src/js/containers/search/visualizations/rank/RankVisualizationWrapperContainer.jsx
+++ b/src/js/containers/search/visualizations/rank/RankVisualizationWrapperContainer.jsx
@@ -198,7 +198,7 @@ export class RankVisualizationWrapperContainer extends React.Component {
             dataSeries.push(result._amount);
 
             if (this.state.scope === 'recipient_duns' && !this.props.subaward) {
-                const recipientLink = result.recipientId ? `recipient/${result.recipientId}` : '';
+                const recipientLink = result.recipientId ? `recipient/${result.recipientId}/latest` : '';
                 linkSeries.push(recipientLink);
             }
 
diff --git a/tests/containers/covid19/SpendingByRecipientContainer-test.jsx b/tests/containers/covid19/SpendingByRecipientContainer-test.jsx
index 58280e630e..fe41b10241 100644
--- a/tests/containers/covid19/SpendingByRecipientContainer-test.jsx
+++ b/tests/containers/covid19/SpendingByRecipientContainer-test.jsx
@@ -35,10 +35,10 @@ describe('SpendingByRecipientContainer', () => {
                     (
                         <>
                             {mockResults[0].description}&nbsp;(
-                            <Link to="/recipient/d2894d22-67fc-f9cb-4005-33fa6a29ef86-C">
+                            <Link to="/recipient/d2894d22-67fc-f9cb-4005-33fa6a29ef86-C/latest">
                                 as Child
                             </Link>,&nbsp;
-                            <Link to="/recipient/d2894d22-67fc-f9cb-4005-33fa6a29ef86-R">
+                            <Link to="/recipient/d2894d22-67fc-f9cb-4005-33fa6a29ef86-R/latest">
                                 as Recipient
                             </Link>
                             )
@@ -50,7 +50,7 @@ describe('SpendingByRecipientContainer', () => {
                 ],
                 [
                     (
-                        <Link to="/recipient/3c92491a-f2cd-ec7d-294b-7daf91511866-R">
+                        <Link to="/recipient/3c92491a-f2cd-ec7d-294b-7daf91511866-R/latest">
                             RECIPIENT 2
                         </Link>
                     ),
@@ -92,10 +92,10 @@ describe('SpendingByRecipientContainer', () => {
                     (
                         <>
                             {mockLoanResults[0].description}&nbsp;(
-                            <Link to="/recipient/d2894d22-67fc-f9cb-4005-33fa6a29ef86-C">
+                            <Link to="/recipient/d2894d22-67fc-f9cb-4005-33fa6a29ef86-C/latest">
                                 as Child
                             </Link>,&nbsp;
-                            <Link to="/recipient/d2894d22-67fc-f9cb-4005-33fa6a29ef86-R">
+                            <Link to="/recipient/d2894d22-67fc-f9cb-4005-33fa6a29ef86-R/latest">
                                 as Recipient
                             </Link>
                             )
@@ -108,7 +108,7 @@ describe('SpendingByRecipientContainer', () => {
                 ],
                 [
                     (
-                        <Link to="/recipient/3c92491a-f2cd-ec7d-294b-7daf91511866-R">
+                        <Link to="/recipient/3c92491a-f2cd-ec7d-294b-7daf91511866-R/latest">
                             RECIPIENT 2
                         </Link>
                     ),

From 41b277e2790ff500abc1d88914e1202c33289bd5 Mon Sep 17 00:00:00 2001
From: Maxwell Kendall <maxwell.n.kendall@gmail.com>
Date: Wed, 17 Mar 2021 11:22:49 -0400
Subject: [PATCH 5/7] [DEV-6937] removing troubleshooting changes from sandbox
 testing

---
 scripts/sitemaps/buildSitemap.js | 22 ++++++++++------------
 scripts/sitemaps/pages.js        |  2 +-
 scripts/sitemaps/robots.txt      |  0
 3 files changed, 11 insertions(+), 13 deletions(-)
 delete mode 100644 scripts/sitemaps/robots.txt

diff --git a/scripts/sitemaps/buildSitemap.js b/scripts/sitemaps/buildSitemap.js
index 000c0a5726..2813c15fbb 100644
--- a/scripts/sitemaps/buildSitemap.js
+++ b/scripts/sitemaps/buildSitemap.js
@@ -219,15 +219,13 @@ const createSitemapDirectory = () => new Promise(
         })
 );
 
-createRobots();
-
-// createSitemapDirectory()
-//     .then(() => buildIndividualSitemaps())
-//     .then((individalSiteMaps) => {
-//         buildIndexedSitemap(individalSiteMaps);
-//         createRobots();
-//     })
-//     .catch((e) => {
-//         console.log(`error build site maps: ${e}`);
-//         throw e.message;
-//     });
+createSitemapDirectory()
+    .then(() => buildIndividualSitemaps())
+    .then((individalSiteMaps) => {
+        buildIndexedSitemap(individalSiteMaps);
+        createRobots();
+    })
+    .catch((e) => {
+        console.log(`error build site maps: ${e}`);
+        throw e.message;
+    });
diff --git a/scripts/sitemaps/pages.js b/scripts/sitemaps/pages.js
index 9d0738ac33..a1f8ee91c3 100644
--- a/scripts/sitemaps/pages.js
+++ b/scripts/sitemaps/pages.js
@@ -1,7 +1,7 @@
 const moment = require('moment');
 
 const apiUrl = process.env.API_URL || 'https://api.usaspending.gov:443';
-const siteUrl = 'https://sandbox.usaspending.gov';
+const siteUrl = process.env.SITE_URL || 'https://www.usaspending.gov';
 
 const routes = [
     // no longer importing these from RouterRoutes because the many import statements in that file completely mess up babel
diff --git a/scripts/sitemaps/robots.txt b/scripts/sitemaps/robots.txt
deleted file mode 100644
index e69de29bb2..0000000000

From cb3dac4e57fb4f98576f8edf995b591f9e194e8e Mon Sep 17 00:00:00 2001
From: Maxwell Kendall <maxwell.n.kendall@gmail.com>
Date: Wed, 17 Mar 2021 10:22:30 -0400
Subject: [PATCH 6/7] [DEV-6881] fixing time-sensitive broken test.

Instead of checking the timezone offset value which changes based
on whether or not we're in DST, we check for the string value of the
current timezone, which is America/New York.
---
 tests/helpers/aboutTheDataHelper-test.js | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/helpers/aboutTheDataHelper-test.js b/tests/helpers/aboutTheDataHelper-test.js
index 1176cf9501..559b1436ed 100644
--- a/tests/helpers/aboutTheDataHelper-test.js
+++ b/tests/helpers/aboutTheDataHelper-test.js
@@ -30,6 +30,8 @@ const mockPeriods = {
     }
 };
 
+const { timeZone } = Intl.DateTimeFormat().resolvedOptions();
+
 describe('About The Data Helper', () => {
     describe('formatPublicationDates', () => {
         const mockData = [
@@ -54,7 +56,7 @@ describe('About The Data Helper', () => {
         });
         it('should format dates if they exist', () => {
             // or, 5hr offset America/New_York UTC-05:000
-            expect(new Date().getTimezoneOffset()).toBe(300);
+            expect(timeZone).toBe('America/New_York');
             expect(data[0][0]).toBe('04/30/2020');
             expect(data[1][0]).toBe('07/31/2020');
             expect(data[1][1]).toBe('08/30/2020');
@@ -70,7 +72,7 @@ describe('About The Data Helper', () => {
         ['publication_date', null, {}, '--'],
         ['certification_date', null, {}, '--']
     ])('renderDeadline: for title %s when deadline is %s returns %s', (title, timestamp, obj, rtrn) => {
-        expect(new Date().getTimezoneOffset()).toBe(300);
+        expect(timeZone).toBe('America/New_York');
         if (title === 'publication_date') {
             expect(renderDeadline(title, obj)).toEqual(rtrn);
         }

From 406fc9c4d29362d6aea940d83782c151fb053e3f Mon Sep 17 00:00:00 2001
From: Maxwell Kendall <maxwell.n.kendall@gmail.com>
Date: Thu, 18 Mar 2021 12:26:03 -0400
Subject: [PATCH 7/7] reverting #2437 from DEV-6973 due to bad timing

---
 .../components/covid19/DataSourcesAndMethodologiesPage.jsx   | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/js/components/covid19/DataSourcesAndMethodologiesPage.jsx b/src/js/components/covid19/DataSourcesAndMethodologiesPage.jsx
index 1171882bb6..b007ee1df7 100644
--- a/src/js/components/covid19/DataSourcesAndMethodologiesPage.jsx
+++ b/src/js/components/covid19/DataSourcesAndMethodologiesPage.jsx
@@ -691,10 +691,7 @@ export default () => {
                                             For <strong>Award Spending (including Loan Spending)</strong>, sum together:
                                             <ul>
                                                 <li>
-                                                    Outlayed Amount Funded by COVID-19 Supplementals for every award ID (award unique key) tagged with a COVID-19 DEFC [for Linked Awards only].
-                                                </li>
-                                                <li>
-                                                    Gross Outlay Amount, Downward Adjustments of Prior Year Prepaid Advanced Undelivered Orders and Obligation Refunds Collected, and Downward Adjustments of Prior Year Paid Delivered Orders and Obligations Refunds Collected for every award ID (award unique key) tagged with a COVID-19 DEFC [for Linked and Unlinked Awards].
+                                                    Either Gross Outlay Amount (for linked and unlinked data) or Outlayed Amount Funded by COVID-19 Supplementals (for linked data only) for every award ID (award unique key) tagged with a COVID-19 DEFC.
                                                 </li>
                                                 <li>
                                                     Filter by any award type as desired.