diff --git a/notebooks/exploration.ipynb b/notebooks/exploration.ipynb index 1aa88a8..059523d 100644 --- a/notebooks/exploration.ipynb +++ b/notebooks/exploration.ipynb @@ -2,25 +2,283 @@ "cells": [ { "cell_type": "code", - "execution_count": 410, + "execution_count": 47, "id": "98ded03d-0208-4416-a5e5-720a2e0742fa", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
titleurlbrandmain_imageskudescriptionraw_descriptiongtin13currencyprice...sub_category_1sub_category_2sub_category_3imagesraw_specificationsspecificationshighlightsraw_highlightsuniq_idscraped_at
0NMR Distribution Zoltar Fortunes Playing Cards...https://www.target.com/p/nmr-distribution-zolt...NMR Distributionhttps://target.scene7.com/is/image/Target/GUES...81917300Zoltar the Great Gypsy can see your future… Yo...<div class=\"h-margin-v-default\" data-test=\"ite...840391145528USD10.99...GamesAdult GamesNaNhttps://target.scene7.com/is/image/Target/GUES...<div class=\"styles__StyledCol-sc-ct8kx6-0 iKGd...Suggested Age: 6 Years and Up | Suggested Age:...ZOLTAR CARDS: Zoltar the Great Gypsy can see y...<li class=\"styles__Bullet-sc-6aebpn-0 eIfLaI\">...898ca3c8-8bfa-5fac-a48e-53879845cf4806/12/22
1The Gifted Stationery 2021 - 2022 Monthly Wall...https://www.target.com/p/the-gifted-stationery...The Gifted Stationaryhttps://target.scene7.com/is/image/Target/GUES...8482100716 month wall calendar provides easy planning ...<div class=\"h-margin-v-default\" data-test=\"ite...9781801433983USD12.99...CalendarsNaNNaNhttps://target.scene7.com/is/image/Target/GUES...<div class=\"styles__StyledCol-sc-ct8kx6-0 iKGd...Dimensions (Overall): 11.9 inches (L), 11.9 in...16-MONTH CALENDAR: Easy planning and goal sett...<li class=\"styles__Bullet-sc-6aebpn-0 eIfLaI\">...b7c8e8ce-55b6-529d-b52e-dbfbd70b066e06/12/22
2Doctor Who: Series Three, Part Two (DVD)https://www.target.com/p/doctor-who-series-thr...Warner Bros.https://target.scene7.com/is/image/Target/GUES...15432753Tenth Doctor David Tennant (Broadchurch, Harry...<div class=\"h-margin-v-default\" data-test=\"ite...883929408115USD10.89...MoviesNaNNaNhttps://target.scene7.com/is/image/Target/GUES...<div class=\"styles__StyledCol-sc-ct8kx6-0 iKGd...Movie Category: Television | Movie Category: T...Run Time: 315:00 min | Disc Count: 2 | Rating:...<li class=\"styles__Bullet-sc-6aebpn-0 eIfLaI\">...97fd83d8-e76b-5bac-999c-b734fdcafe6606/12/22
3Blue Panda 75 Pieces Tie Dye Birthday Party Su...https://www.target.com/p/blue-panda-75-pieces-...Blue Pandahttps://target.scene7.com/is/image/Target/GUES...84199597Throwing a party has never been easier, we’ve ...<div class=\"h-margin-v-default\" data-test=\"ite...194425194489USD26.99...Birthday Party SuppliesNaNNaNhttps://target.scene7.com/is/image/Target/GUES...<div class=\"styles__StyledCol-sc-ct8kx6-0 iKGd...Number of Pieces: 24 | Number of Pieces: 24 | ...Serves 24: Includes 1 plastic table cover, a b...<li class=\"styles__Bullet-sc-6aebpn-0 eIfLaI\">...3b4d60b0-0888-596e-babe-d9e6e4e3131c06/12/22
4Sullivans Hammered Metal Wall Medallions Set o...https://www.target.com/p/sullivans-hammered-me...Sullivanshttps://target.scene7.com/is/image/Target/GUES...86345566Bring a little charm and delight to your space...<div class=\"h-margin-v-default\" data-test=\"ite...23271231140USD118.99...Home DecorWall DecorWall Accentshttps://target.scene7.com/is/image/Target/GUES...<div class=\"styles__StyledCol-sc-ct8kx6-0 iKGd...Dimensions (Overall): 23 inches (H) x 1 inches...This unique wall set will be the perfect addit...<li class=\"styles__Bullet-sc-6aebpn-0 eIfLaI\">...036188d9-4546-5952-af8f-bc234d1f811306/12/22
\n", + "

5 rows × 24 columns

\n", + "
" + ], + "text/plain": [ + " title \\\n", + "0 NMR Distribution Zoltar Fortunes Playing Cards... \n", + "1 The Gifted Stationery 2021 - 2022 Monthly Wall... \n", + "2 Doctor Who: Series Three, Part Two (DVD) \n", + "3 Blue Panda 75 Pieces Tie Dye Birthday Party Su... \n", + "4 Sullivans Hammered Metal Wall Medallions Set o... \n", + "\n", + " url brand \\\n", + "0 https://www.target.com/p/nmr-distribution-zolt... NMR Distribution \n", + "1 https://www.target.com/p/the-gifted-stationery... The Gifted Stationary \n", + "2 https://www.target.com/p/doctor-who-series-thr... Warner Bros. \n", + "3 https://www.target.com/p/blue-panda-75-pieces-... Blue Panda \n", + "4 https://www.target.com/p/sullivans-hammered-me... Sullivans \n", + "\n", + " main_image sku \\\n", + "0 https://target.scene7.com/is/image/Target/GUES... 81917300 \n", + "1 https://target.scene7.com/is/image/Target/GUES... 84821007 \n", + "2 https://target.scene7.com/is/image/Target/GUES... 15432753 \n", + "3 https://target.scene7.com/is/image/Target/GUES... 84199597 \n", + "4 https://target.scene7.com/is/image/Target/GUES... 86345566 \n", + "\n", + " description \\\n", + "0 Zoltar the Great Gypsy can see your future… Yo... \n", + "1 16 month wall calendar provides easy planning ... \n", + "2 Tenth Doctor David Tennant (Broadchurch, Harry... \n", + "3 Throwing a party has never been easier, we’ve ... \n", + "4 Bring a little charm and delight to your space... \n", + "\n", + " raw_description gtin13 currency \\\n", + "0
... \n", + "1
  • ... \n", + "2
  • ... \n", + "3
  • ... \n", + "4
  • ... \n", + "\n", + " uniq_id scraped_at \n", + "0 898ca3c8-8bfa-5fac-a48e-53879845cf48 06/12/22 \n", + "1 b7c8e8ce-55b6-529d-b52e-dbfbd70b066e 06/12/22 \n", + "2 97fd83d8-e76b-5bac-999c-b734fdcafe66 06/12/22 \n", + "3 3b4d60b0-0888-596e-babe-d9e6e4e3131c 06/12/22 \n", + "4 036188d9-4546-5952-af8f-bc234d1f8113 06/12/22 \n", + "\n", + "[5 rows x 24 columns]" + ] + }, + "execution_count": 47, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import functools\n", "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline\n", "from IPython.display import display, HTML\n", "\n", "in_file = \"/home/jovyan/data/large_target_store_products_dataset_sample - large_target_store_products_dataset_sample.csv\"\n", "data = pd.read_csv(in_file)\n", - "\n", - "def look_for_matches(data: pd.DataFrame, pattern: str, colname : str = \"raw_specifications\") -> str:\n", - " \"\"\"Useful for finding cells in raw_specifications containing a given string\"\"\"\n", - " return data.loc[data.loc[:, colname].str.contains(pattern), colname].iloc[0]\n", - "\n", - "def render_html(html: str):\n", - " \"\"\"Render an html string\"\"\"\n", - " display(HTML(html))" + "data.head()" ] }, { @@ -52,7 +310,7 @@ }, { "cell_type": "code", - "execution_count": 132, + "execution_count": 3, "id": "165723d1-8152-4e30-b25a-cbca7f4935f9", "metadata": {}, "outputs": [ @@ -70,12 +328,16 @@ } ], "source": [ + "def render_html(html: str):\n", + " \"\"\"Render an html string\"\"\"\n", + " display(HTML(html))\n", + "\n", "render_html(data.loc[0, 'raw_specifications'])" ] }, { "cell_type": "code", - "execution_count": 134, + "execution_count": 5, "id": "d153a732-f18c-4fb7-9282-9c02c49c3cc0", "metadata": {}, "outputs": [ @@ -93,12 +355,16 @@ } ], "source": [ + "def look_for_matches(data: pd.DataFrame, pattern: str, colname : str = \"raw_specifications\") -> str:\n", + " \"\"\"Useful for finding cells in raw_specifications containing a given string\"\"\"\n", + " return data.loc[data.loc[:, colname].str.contains(pattern), colname].iloc[0]\n", + "\n", "render_html(look_for_matches(data, \"Material\"))" ] }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 6, "id": "b06e66af-e0e3-466c-8797-546d5e076f32", "metadata": {}, "outputs": [ @@ -121,7 +387,7 @@ }, { "cell_type": "code", - "execution_count": 137, + "execution_count": 7, "id": "015a0d99-b4c2-47cf-9e23-59485a450470", "metadata": {}, "outputs": [ @@ -144,7 +410,7 @@ }, { "cell_type": "code", - "execution_count": 138, + "execution_count": 8, "id": "99cbfba8-4324-4d2a-b2e2-d766a5e57964", "metadata": {}, "outputs": [ @@ -167,7 +433,7 @@ }, { "cell_type": "code", - "execution_count": 139, + "execution_count": 9, "id": "6c05e1db-7181-422e-924b-e67a28886abe", "metadata": {}, "outputs": [ @@ -190,7 +456,7 @@ }, { "cell_type": "code", - "execution_count": 140, + "execution_count": 10, "id": "bf3a07d9-11c0-48c4-be03-373a3ba5f755", "metadata": {}, "outputs": [ @@ -213,183 +479,7 @@ }, { "cell_type": "code", - "execution_count": 31, - "id": "ffdb3244-a20d-427b-9c23-4d8c6cf4ce6d", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Material\n", - "raw_specifications\n", - "True 105\n", - "False 62\n", - "Name: count, dtype: int64\n", - "\n", - "Package Quantity\n", - "raw_specifications\n", - "False 143\n", - "True 24\n", - "Name: count, dtype: int64\n", - "\n", - "Weight\n", - "raw_specifications\n", - "False 106\n", - "True 61\n", - "Name: count, dtype: int64\n", - "\n", - "Dimensions\n", - "raw_specifications\n", - "True 108\n", - "False 59\n", - "Name: count, dtype: int64\n", - "\n", - "TCIN\n", - "raw_specifications\n", - "True 167\n", - "Name: count, dtype: int64\n", - "\n", - "Origin\n", - "raw_specifications\n", - "True 167\n", - "Name: count, dtype: int64\n", - "\n" - ] - } - ], - "source": [ - "# let's see how many of the specifications are there\n", - "\n", - "colname = \"raw_specifications\"\n", - "patterns = [\"Material\", \"Package Quantity\", \"Weight\", \"Dimensions\", \"TCIN\", \"Origin\"]\n", - "\n", - "for pattern in patterns:\n", - " print(pattern)\n", - " print(data.loc[:, colname].str.contains(pattern).value_counts())\n", - " print()" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "dc44956b-ea8c-44ce-8bc5-e0472273e7a3", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "raw_specifications\n", - "False 162\n", - "True 5\n", - "Name: count, dtype: int64" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# let's see how many of the specifications are there\n", - "\n", - "colname = \"raw_specifications\"\n", - "patterns = [\"Material\", \"Package Quantity\", \"Weight\", \"Dimensions\", \"TCIN\", \"Origin\"]\n", - "\n", - "sels = [data.loc[:, colname].str.contains(pattern) for pattern in patterns]\n", - "sel = functools.reduce(lambda x, y: x & y, sels)\n", - "\n", - "sel.value_counts()" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "id": "107da59a-4bd8-4b0c-9e6e-95f38788eaf3", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "

    Specifications

    Dimensions (Overall): .1 inches (H) x 6.0 inches (W) x 13.5 inches (D)
    Weight: .35 ounces
    Package Quantity: 3
    Material: Wood
    TCIN: 82555842
    UPC: 843128185798
    Origin: imported
    WARNING:⚠ This product can expose you to chemicals including Formaldehyde (gas), which is known to the State of California to cause cancer and birth defects or other reproductive harm. For more information, go to www.P65Warnings.ca.gov www.p65warnings.ca.gov

    The above item details were provided by the Target Plus™ Partner. Target does not represent or warrant that this information is accurate or complete. On occasion, manufacturers may modify their items and update their labels.

    We recommend that you do not rely solely on the information presented. If you have a specific question about this item, you may consult the item's label, contact the manufacturer directly or call Target Guest Services at 1-800-591-3869.

    " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "

    Specifications

    Dimensions (Overall): 9.02 inches (L), 4.88 inches (H) x 5.0 inches (W)
    Weight: 1.63 pounds
    Package Quantity: 100
    Material: Paper
    TCIN: 84236733
    UPC: 194425198586
    Origin: imported

    The above item details were provided by the Target Plus™ Partner. Target does not represent or warrant that this information is accurate or complete. On occasion, manufacturers may modify their items and update their labels.

    We recommend that you do not rely solely on the information presented. If you have a specific question about this item, you may consult the item's label, contact the manufacturer directly or call Target Guest Services at 1-800-591-3869.

    " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "

    Specifications

    Dimensions (Overall): 1.14 inches (H) x 5.04 inches (W) x 6.3 inches (D)
    Weight: .15 ounces
    Package Quantity: 12
    Material: Wood
    TCIN: 82021299
    UPC: 194425108806
    Origin: imported

    The above item details were provided by the Target Plus™ Partner. Target does not represent or warrant that this information is accurate or complete. On occasion, manufacturers may modify their items and update their labels.

    We recommend that you do not rely solely on the information presented. If you have a specific question about this item, you may consult the item's label, contact the manufacturer directly or call Target Guest Services at 1-800-591-3869.

    " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "

    Specifications

    Dimensions (Overall): 9.79 inches (L), 4.13 inches (W)
    Weight: 2.61 ounces
    Closure Type: Flap
    Package Quantity: 250
    Material: Paper
    TCIN: 81501830
    UPC: 083514896277
    Origin: made in the USA

    The above item details were provided by the Target Plus™ Partner. Target does not represent or warrant that this information is accurate or complete. On occasion, manufacturers may modify their items and update their labels.

    We recommend that you do not rely solely on the information presented. If you have a specific question about this item, you may consult the item's label, contact the manufacturer directly or call Target Guest Services at 1-800-591-3869.

    " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "

    Specifications

    Dimensions (Overall): 2.24 inches (L), 6.0 inches (W)
    Weight: 1.6 ounces
    Closure Type: Flap
    Package Quantity: 100
    Material: Cardboard
    TCIN: 81843685
    UPC: 087547421000
    Origin: imported

    The above item details were provided by the Target Plus™ Partner. Target does not represent or warrant that this information is accurate or complete. On occasion, manufacturers may modify their items and update their labels.

    We recommend that you do not rely solely on the information presented. If you have a specific question about this item, you may consult the item's label, contact the manufacturer directly or call Target Guest Services at 1-800-591-3869.

    " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "https://www.target.com/p/genie-crafts-wood-quote-signs-eat-drink-love-wood-letter-signs-drawing-stencils-wall-decor/-/A-82555842\n", - "https://www.target.com/p/sparkle-and-bash-100-pack-gold-foil-initial-letter-k-white-monogram-paper-napkins-for-dinner-party-4-x-8-in/-/A-84236733\n", - "https://www.target.com/p/wooden-rectangles-for-crafts-panel-board-4-x-6-in-12-pack/-/A-82021299\n", - "https://www.target.com/p/southworth-25-cotton-10-business-envelope-ivory-24-lbs-wove-250-box-fsc-j404i10/-/A-81501830\n", - "https://www.target.com/p/universal-self-seal-catalog-envelope-6-x-9-white-100-box-42100/-/A-81843685\n" - ] - } - ], - "source": [ - "for x in data.loc[sel, \"raw_specifications\"]:\n", - " render_html(x)\n", - "for x in data.loc[sel, \"url\"]:\n", - " print(x)" - ] - }, - { - "cell_type": "code", - "execution_count": 67, + "execution_count": 11, "id": "3853dd07-1977-44c6-bc97-2ecdaa98980e", "metadata": {}, "outputs": [ @@ -402,7 +492,7 @@ "Name: count, dtype: int64" ] }, - "execution_count": 67, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -418,7 +508,7 @@ }, { "cell_type": "code", - "execution_count": 86, + "execution_count": 12, "id": "be8a7628-9199-4f25-a533-d041a7ff540c", "metadata": {}, "outputs": [ @@ -613,7 +703,7 @@ "[3 rows x 24 columns]" ] }, - "execution_count": 86, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -633,17 +723,119 @@ }, { "cell_type": "code", - "execution_count": 359, + "execution_count": 13, "id": "3fd14c61-c1f8-4b18-b04a-9b616e6cf9a7", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "\"Dimensions (Overall): 11.9 inches (L), 11.9 inches (W) | Dimensions (Overall): 11.9 inches (L), 11.9 inches (W) | Dated format: Monthly | Dated format: Monthly | Calendar year: 2022 | Calendar year: 2022 | Material: Paper | Material: Paper | TCIN: 84821007 | UPC: 9781801433983 | Origin: imported | The above item details were provided by the Target Plus™ Partner. Target does not represent or warrant that this information is accurate or complete. On occasion, manufacturers may modify their items and update their labels.We recommend that you do not rely solely on the information presented. If you have a specific question about this item, you may consult the item's label, contact the manufacturer directly or call Target Guest Services at 1-800-591-3869.\"" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ + "import xml.etree.ElementTree as ET\n", + "\n", "# There seem to be dupliucates on \"specifications\" that are not \n", "# found on \"raw_specifications\"\n", "# I think it's safe to just remove the duplicates\n", "\n", "specifications = data.loc[1,\"specifications\"]\n", - "\n", + "specifications\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "88894d7c-dc68-42ee-9c9d-a53474762a7a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 None\n", + "1 Paper\n", + "2 None\n", + "3 Paper\n", + "4 Metal\n", + " ... \n", + "162 Plastic\n", + "163 None\n", + "164 Fabric\n", + "165 None\n", + "166 Stoneware\n", + "Name: raw_specifications, Length: 167, dtype: object\n", + "0 None\n", + "1 None\n", + "2 None\n", + "3 24\n", + "4 None\n", + " ... \n", + "162 None\n", + "163 None\n", + "164 None\n", + "165 None\n", + "166 1\n", + "Name: raw_specifications, Length: 167, dtype: object\n", + "0 None\n", + "1 11.9 inches (L), 11.9 inches (W)\n", + "2 None\n", + "3 None\n", + "4 23 inches (H) x 1 inches (W) x 23 inches (D)\n", + " ... \n", + "162 5.0 inches (W) x 9.5 inches (D)\n", + "163 12.0 inches (H) x 12.0 inches (W)\n", + "164 4.75 inches (H) x 2.25 inches (W) x 2.25 inch...\n", + "165 None\n", + "166 4.5 inches (H) x 12.5 inches (W) x 4.5 inches...\n", + "Name: raw_specifications, Length: 167, dtype: object\n", + "0 None\n", + "1 None\n", + "2 None\n", + "3 None\n", + "4 4.65 pounds\n", + " ... \n", + "162 .28 pounds\n", + "163 32.0 pounds\n", + "164 None\n", + "165 None\n", + "166 None\n", + "Name: raw_specifications, Length: 167, dtype: object\n", + "0 81917300\n", + "1 84821007\n", + "2 15432753\n", + "3 84199597\n", + "4 86345566\n", + " ... \n", + "162 83388852\n", + "163 80836585\n", + "164 75477923\n", + "165 85634544\n", + "166 80239765\n", + "Name: raw_specifications, Length: 167, dtype: object\n", + "0 imported\n", + "1 imported\n", + "2 Made in the USA\n", + "3 imported\n", + "4 imported\n", + " ... \n", + "162 made in the USA or imported\n", + "163 made in the USA or imported\n", + "164 made in the USA or imported\n", + "165 imported\n", + "166 imported\n", + "Name: raw_specifications, Length: 167, dtype: object\n" + ] + } + ], + "source": [ "def parse_specs(specifications: str) -> dict[str,str]:\n", " fields_mapping = {\n", " \"Material\": \"materials\",\n", @@ -721,14 +913,12 @@ " return parse_raw_specs(specs).get(value)\n", "\n", "for x in [\"materials\", \"packaging\", \"dimensions\", \"weight\", \"tcin\", \"origin\"]:\n", - " data[x] = data[\"raw_specifications\"].apply(parse_value, value=x)\n", - "\n", - "# set(data[\"material\"])\n" + " print(data[\"raw_specifications\"].apply(parse_value, value=x))\n" ] }, { "cell_type": "code", - "execution_count": 305, + "execution_count": 17, "id": "5902f1b3-a77e-4353-92aa-71cc729e8c87", "metadata": {}, "outputs": [ @@ -746,10 +936,10 @@ "164 {'height': 12.065, 'width': 5.715, 'depth': 5....\n", "165 None\n", "166 {'height': 11.43, 'width': 31.75, 'depth': 11.43}\n", - "Name: dimensions, Length: 167, dtype: object" + "Name: raw_specifications, Length: 167, dtype: object" ] }, - "execution_count": 305, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } @@ -758,8 +948,6 @@ "import re\n", "from typing import Optional\n", "\n", - "dimensions = \"16 inches (H) x 23 inches (W) x 23 inches (D)\"\n", - "\n", "def parse_dimensions_measure(dimensions: str, measure: str) -> Optional[dict[str,str]]:\n", " expr = f\"(?P\\d*[.,]?\\d*)\\s+(?P[a-zA-Z]*)\\s+\\({measure}\\)\" \n", " if match := re.search(expr, dimensions):\n", @@ -794,16 +982,38 @@ " if value is not None\n", " }\n", "\n", - "dimensions = data[\"dimensions\"].apply(parse_dimensions)\n", - "dimensions\n" + "dimensions = data[\"raw_specifications\"].apply(parse_value, value=\"dimensions\").apply(parse_dimensions)\n", + "dimensions" ] }, { "cell_type": "code", - "execution_count": 341, + "execution_count": 51, "id": "26ece1d2-c466-498c-aeb6-5610fb16c7d3", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0 NaN\n", + "1 NaN\n", + "2 NaN\n", + "3 NaN\n", + "4 2109.202800\n", + " ... \n", + "162 127.005760\n", + "163 14514.944000\n", + "164 0.078812\n", + "165 NaN\n", + "166 0.829595\n", + "Name: raw_specifications, Length: 167, dtype: float64" + ] + }, + "execution_count": 51, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "def units_to_g(value: float, unit: str) -> float:\n", " conversions = {\n", @@ -845,19 +1055,44 @@ " \n", " return height * width * depth / 5000\n", "\n", - "weight = data[\"weight\"].apply(parse_weight)\n", - "# weight\n", + "weight = data[\"raw_specifications\"].apply(parse_value, value=\"weight\").apply(parse_weight)\n", "\n", "dimensional_weight = list(map(calculate_dimensional_weight, dimensions))\n", - "# dimensional_weight\n", "\n", "sel = weight.isnull()\n", - "weight[sel] = pd.Series(dimensional_weight)[sel]" + "weight[sel] = pd.Series(dimensional_weight)[sel]\n", + "\n", + "weight" ] }, { "cell_type": "code", - "execution_count": 404, + "execution_count": 74, + "id": "49c7495f-1f36-4dc9-8686-de3af7b230a9", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGdCAYAAACyzRGfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAxjklEQVR4nO3de3BUZZ7/8U9jQidgiFwknSwEwhojEGEwsBAUE0XCgLK6uDuOIsKOTslwN0WlDPxhcEqCFMNGFoVBkcuyiDsbUKZATKaGBBVQCKHkZsQ1QzKYmI0LCdeEy/P7w0n/aHKBTrqfpOP7VXWqPOc855zv04d+/OT0Od0OY4wRAACAJR1auwAAAPDTQvgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYFVQaxdwo2vXrum7775TWFiYHA5Ha5cD/CQZY3T27FlFRUWpQ4fA+BuFsQNoXd6MG20ufHz33Xfq3bt3a5cBQFJpaal69erV2mXcEsYOoG24lXGjzYWPsLAwST8W36VLl1auBvhpqq6uVu/evd3vx0DA2AG0Lm/GjTYXPuoul3bp0oUBBGhlgfTxBWMH0DbcyrgRGB/mAgCAdoPwAQAArCJ8AAAAqwgfAADAKq/Dx6lTp/Tss8+qe/fu6tSpk372s5+poKDAvd4Yo4yMDEVFRSk0NFTJyck6evSoT4sGAACBy6vwcfr0ad1///0KDg7WRx99pGPHjul3v/ud7rjjDnebJUuWaNmyZVqxYoX2798vl8ulMWPG6OzZs76uHQAABCCvHrV9/fXX1bt3b61du9a9rG/fvu7/NsYoKytLCxYs0MSJEyVJ69evV0REhDZt2qQXX3zRN1UDAICA5dWVj23btmno0KH6l3/5F/Xs2VNDhgzR22+/7V5fXFys8vJypaSkuJc5nU4lJSVpz549De6zpqZG1dXVHhMAAGi/vAof3377rVauXKnY2Fh9/PHHmjZtmmbPnq0NGzZIksrLyyVJERERHttFRES4190oMzNT4eHh7omvRwYAoH3zKnxcu3ZN9913nxYtWqQhQ4boxRdf1K9//WutXLnSo92N325mjGn0G8/S09NVVVXlnkpLS73sAgAACCRehY/IyEgNGDDAY1n//v1VUlIiSXK5XJJU7ypHRUVFvashdZxOp/vrkPlaZCDwrVy5UoMGDXK/nxMTE/XRRx81uU1+fr4SEhIUEhKifv36adWqVZaqBdAavAof999/v4qKijyWff311+rTp48kKSYmRi6XS7m5ue71tbW1ys/P18iRI31QLoC2rlevXlq8eLEOHDigAwcO6OGHH9bjjz/e6CP3xcXFGj9+vEaNGqXCwkLNnz9fs2fPVnZ2tuXKAVhjvPDFF1+YoKAg89prr5kTJ06Y//zP/zSdOnUyGzdudLdZvHixCQ8PN1u2bDGHDx82Tz/9tImMjDTV1dW3dIyqqiojyVRVVXlTGgAf8vX7sGvXruadd95pcF1aWpq55557PJa9+OKLZsSIEV4dg7EDaF3evAe9etR22LBh2rp1q9LT0/Xqq68qJiZGWVlZmjRpkrtNWlqaLl68qOnTp+v06dMaPny4cnJy/PbT3CUlJaqsrHTP9+jRQ9HR0X45FgDvXL16VX/4wx90/vx5JSYmNthm7969Hk/ISdLYsWO1Zs0aXb58WcHBwQ1uV1NTo5qaGve8L5+Uu3FckRhbAF/yKnxI0mOPPabHHnus0fUOh0MZGRnKyMhoSV23pKSkRHFx/XXp0gX3spCQTioqOs4gAbSiw4cPKzExUZcuXdLtt9+urVu31rtfrE55eXmDT8hduXJFlZWVioyMbHC7zMxMLVy40Oe1NzSuSIwtgC8F9G+7VFZW/m2A2CipQNJGXbp0od5fLADsiouL06FDh7Rv3z795je/0ZQpU3Ts2LFG2zf0hFxDy6/nryfl6o8rjC2Ar3l95aNt6i/pvtYuAsDfdOzYUXfddZckaejQodq/f7/eeOMN/f73v6/X1uVyNfiEXFBQkLp3797oMZxOp5xOp28L98C4AvhLQF/5ABAYjDEe92dcLzEx0eMJOUnKycnR0KFDG73fA0BgI3wA8Kn58+frk08+0V/+8hcdPnxYCxYsUF5envvG9PT0dD333HPu9tOmTdPJkyeVmpqq48eP691339WaNWs0b9681uoCAD9rJx+7AGgrvv/+e02ePFllZWUKDw/XoEGDtHPnTo0ZM0aSVFZW5v5iQunH7wfasWOHXnrpJb355puKiorS8uXL9eSTT7ZWFwD4GeEDgE+tWbOmyfXr1q2rtywpKUkHDx70U0UA2ho+dgEAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABgFeEDAABYRfgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABgFeEDAABYRfgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABgFeEDAABYRfgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABglVfhIyMjQw6Hw2NyuVzu9cYYZWRkKCoqSqGhoUpOTtbRo0d9XjQAAAhcXl/5GDhwoMrKytzT4cOH3euWLFmiZcuWacWKFdq/f79cLpfGjBmjs2fP+rRoAAAQuLwOH0FBQXK5XO7pzjvvlPTjVY+srCwtWLBAEydOVHx8vNavX68LFy5o06ZNPi8cAAAEJq/Dx4kTJxQVFaWYmBj98pe/1LfffitJKi4uVnl5uVJSUtxtnU6nkpKStGfPnkb3V1NTo+rqao8JAAC0X16Fj+HDh2vDhg36+OOP9fbbb6u8vFwjR47UDz/8oPLycklSRESExzYRERHudQ3JzMxUeHi4e+rdu3czugEAAAKFV+Fj3LhxevLJJ3XvvffqkUce0fbt2yVJ69evd7dxOBwe2xhj6i27Xnp6uqqqqtxTaWmpNyUBaGMyMzM1bNgwhYWFqWfPnnriiSdUVFTU5DZ5eXn1bmZ3OBz66quvLFUNwKYWPWrbuXNn3XvvvTpx4oT7qZcbr3JUVFTUuxpyPafTqS5dunhMAAJXfn6+ZsyYoX379ik3N1dXrlxRSkqKzp8/f9Nti4qKPG5oj42NtVAxANuCWrJxTU2Njh8/rlGjRikmJkYul0u5ubkaMmSIJKm2tlb5+fl6/fXXfVIsgLZv586dHvNr165Vz549VVBQoAcffLDJbXv27Kk77rjDj9UBaAu8uvIxb9485efnq7i4WJ9//rn++Z//WdXV1ZoyZYocDofmzp2rRYsWaevWrTpy5IimTp2qTp066ZlnnvFX/QDauKqqKklSt27dbtp2yJAhioyM1OjRo7Vr164m23KzOhC4vLry8de//lVPP/20Kisrdeedd2rEiBHat2+f+vTpI0lKS0vTxYsXNX36dJ0+fVrDhw9XTk6OwsLC/FI8gLbNGKPU1FQ98MADio+Pb7RdZGSkVq9erYSEBNXU1Og//uM/NHr0aOXl5TV6tSQzM1MLFy70V+kA/Mir8LF58+Ym1zscDmVkZCgjI6MlNQFoJ2bOnKkvv/xSn376aZPt4uLiFBcX555PTExUaWmpli5d2mj4SE9PV2pqqnu+urqap+WAAMFvuwDwi1mzZmnbtm3atWuXevXq5fX2I0aM0IkTJxpdz83qQOBq0Q2nAHAjY4xmzZqlrVu3Ki8vTzExMc3aT2FhoSIjI31cHYC2gPABwKdmzJihTZs26cMPP1RYWJj78fvw8HCFhoZK+vEjk1OnTmnDhg2SpKysLPXt21cDBw5UbW2tNm7cqOzsbGVnZ7daPwD4D+EDgE+tXLlSkpScnOyxfO3atZo6daokqaysTCUlJe51tbW1mjdvnk6dOqXQ0FANHDhQ27dv1/jx422VDcAiwgcAnzLG3LTNunXrPObT0tKUlpbmp4oAtDXccAoAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAqhaFj8zMTDkcDs2dO9e9zBijjIwMRUVFKTQ0VMnJyTp69GhL6wQAAO1Es8PH/v37tXr1ag0aNMhj+ZIlS7Rs2TKtWLFC+/fvl8vl0pgxY3T27NkWFwsAAAJfs8LHuXPnNGnSJL399tvq2rWre7kxRllZWVqwYIEmTpyo+Ph4rV+/XhcuXNCmTZt8VjQAAAhczQofM2bM0KOPPqpHHnnEY3lxcbHKy8uVkpLiXuZ0OpWUlKQ9e/a0rFIAANAueB0+Nm/erIMHDyozM7PeuvLycklSRESEx/KIiAj3uhvV1NSourraYwIQuDIzMzVs2DCFhYWpZ8+eeuKJJ1RUVHTT7fLz85WQkKCQkBD169dPq1atslAtgNbgVfgoLS3VnDlztHHjRoWEhDTazuFweMwbY+otq5OZmanw8HD31Lt3b29KAtDG5Ofna8aMGdq3b59yc3N15coVpaSk6Pz5841uU1xcrPHjx2vUqFEqLCzU/PnzNXv2bGVnZ1usHIAtQd40LigoUEVFhRISEtzLrl69qt27d2vFihXuv27Ky8sVGRnpblNRUVHvakid9PR0paamuuerq6sJIEAA27lzp8f82rVr1bNnTxUUFOjBBx9scJtVq1YpOjpaWVlZkqT+/fvrwIEDWrp0qZ588kl/lwzAMq+ufIwePVqHDx/WoUOH3NPQoUM1adIkHTp0SP369ZPL5VJubq57m9raWuXn52vkyJEN7tPpdKpLly4eE4D2o6qqSpLUrVu3Rtvs3bvX414xSRo7dqwOHDigy5cv+7U+APZ5deUjLCxM8fHxHss6d+6s7t27u5fPnTtXixYtUmxsrGJjY7Vo0SJ16tRJzzzzjO+qBhAQjDFKTU3VAw88UG/suF55eXmD94pduXJFlZWVHldS69TU1KimpsY935L7xUpKSlRZWSlJOn78eKPtrl/Xo0cPRUdHN/uYwE+ZV+HjVqSlpenixYuaPn26Tp8+reHDhysnJ0dhYWG+PhSANm7mzJn68ssv9emnn960bUP3ijW0vE5mZqYWLlzY4hpLSkoUF9dfly5daKJVmaQOevbZZ91LQkI6qajoOAEEaIYWf716Xl6e+3Na6ceBIiMjQ2VlZbp06ZLy8/Ob/IsHQPs0a9Ysbdu2Tbt27VKvXr2abOtyueo9EVdRUaGgoCB17969wW3S09NVVVXlnkpLS5tVZ2Vl5d+Cx0ZJBZJ+20CrM5KuXddmoy5duuC+WgLAOz6/8gHgp80Yo1mzZmnr1q3Ky8tTTEzMTbdJTEzUH//4R49lOTk5Gjp0qIKDgxvcxul0yul0+qTmH/WXdJ+kxj92+f9tALQEPywHwKdmzJihjRs3atOmTQoLC1N5ebnKy8t18eJFd5v09HQ999xz7vlp06bp5MmTSk1N1fHjx/Xuu+9qzZo1mjdvXmt0AYCfET4A+NTKlStVVVWl5ORkRUZGuqf333/f3aasrEwlJSXu+ZiYGO3YsUN5eXn62c9+pt/+9rdavnw5j9kC7RQfuwDwqbobRZuybt26esuSkpJ08OBBP1QEoK3hygcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwyqvwsXLlSg0aNEhdunRRly5dlJiYqI8++si93hijjIwMRUVFKTQ0VMnJyTp69KjPiwYAAIHLq/DRq1cvLV68WAcOHNCBAwf08MMP6/HHH3cHjCVLlmjZsmVasWKF9u/fL5fLpTFjxujs2bN+KR4AAAQer8LHhAkTNH78eN199926++679dprr+n222/Xvn37ZIxRVlaWFixYoIkTJyo+Pl7r16/XhQsXtGnTJn/VDwAAAkyz7/m4evWqNm/erPPnzysxMVHFxcUqLy9XSkqKu43T6VRSUpL27NnT6H5qampUXV3tMQEIbLt379aECRMUFRUlh8OhDz74oMn2eXl5cjgc9aavvvrKTsEArPI6fBw+fFi33367nE6npk2bpq1bt2rAgAEqLy+XJEVERHi0j4iIcK9rSGZmpsLDw91T7969vS0JQBtz/vx5DR48WCtWrPBqu6KiIpWVlbmn2NhYP1UIoDUFebtBXFycDh06pDNnzig7O1tTpkxRfn6+e73D4fBob4ypt+x66enpSk1Ndc9XV1cTQIAAN27cOI0bN87r7Xr27Kk77rjD9wUBaFO8vvLRsWNH3XXXXRo6dKgyMzM1ePBgvfHGG3K5XJJU7ypHRUVFvash13M6ne6nZ+omAD9NQ4YMUWRkpEaPHq1du3Y12ZaPbIHA1eLv+TDGqKamRjExMXK5XMrNzXWvq62tVX5+vkaOHNnSwwBoxyIjI7V69WplZ2dry5YtiouL0+jRo7V79+5Gt+EjWyBwefWxy/z58zVu3Dj17t1bZ8+e1ebNm5WXl6edO3fK4XBo7ty5WrRokWJjYxUbG6tFixapU6dOeuaZZ/xVP4B2IC4uTnFxce75xMRElZaWaunSpXrwwQcb3IaPbIHA5VX4+P777zV58mSVlZUpPDxcgwYN0s6dOzVmzBhJUlpami5evKjp06fr9OnTGj58uHJychQWFuaX4gG0XyNGjNDGjRsbXe90OuV0Oi1WBMBXvAofa9asaXK9w+FQRkaGMjIyWlITAKiwsFCRkZGtXQYAP/D6aRcAuJlz587pm2++cc8XFxfr0KFD6tatm6Kjo5Wenq5Tp05pw4YNkqSsrCz17dtXAwcOVG1trTZu3Kjs7GxlZ2e3VhcA+BHhA4DPHThwQA899JB7vu7ejClTpmjdunUqKytTSUmJe31tba3mzZunU6dOKTQ0VAMHDtT27ds1fvx467UD8D/CBwCfS05OljGm0fXr1q3zmE9LS1NaWpqfqwLQVrT4UVsAAABvED4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABgFeEDAABYRfgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABgFeEDAABYRfgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABgFeEDAABYRfgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVV6Fj8zMTA0bNkxhYWHq2bOnnnjiCRUVFXm0McYoIyNDUVFRCg0NVXJyso4ePerTogEAQODyKnzk5+drxowZ2rdvn3Jzc3XlyhWlpKTo/Pnz7jZLlizRsmXLtGLFCu3fv18ul0tjxozR2bNnfV48AAAIPF6Fj507d2rq1KkaOHCgBg8erLVr16qkpEQFBQWSfrzqkZWVpQULFmjixImKj4/X+vXrdeHCBW3atMkvHQDQ9uzevVsTJkxQVFSUHA6HPvjgg5tuk5+fr4SEBIWEhKhfv35atWqV/wsF0CpadM9HVVWVJKlbt26SpOLiYpWXlyslJcXdxul0KikpSXv27GlwHzU1NaqurvaYAAS28+fPa/DgwVqxYsUttS8uLtb48eM1atQoFRYWav78+Zo9e7ays7P9XCmA1hDU3A2NMUpNTdUDDzyg+Ph4SVJ5ebkkKSIiwqNtRESETp482eB+MjMztXDhwuaWAaANGjdunMaNG3fL7VetWqXo6GhlZWVJkvr3768DBw5o6dKlevLJJ/1UJYDW0uzwMXPmTH355Zf69NNP661zOBwe88aYesvqpKenKzU11T1fXV2t3r17N7csAAFo7969HldMJWns2LFas2aNLl++rODg4Hrb1NTUqKamxj3vzVXTkpISVVZWSpKOHz/ezKo9t62pqZHT6Wx0vkePHoqOjm5Wjc3Z3lf78GZ/vj6eP/jjdW2Nc+3rGnxRkzeaFT5mzZqlbdu2affu3erVq5d7ucvlkvTjFZDIyEj38oqKinpXQ+o4nU6PFwzAT095eXmDV0yvXLmiyspKj/GkTnOvmpaUlCgurr8uXbrQ7HqlMkkd9Oyzz1637DZJVxudDwnppKKi47c0gDdUozfb+2of3uzP18fzB3+9rrbPta9r8EVN3vLqng9jjGbOnKktW7boz3/+s2JiYjzWx8TEyOVyKTc3172strZW+fn5GjlypE8KBtA+NXTFtKHlddLT01VVVeWeSktLb+k4lZWVfxtUN0oqkPTbZlR7RtK1G/ZxtYn5jbp06YLHX5Le1ejd9r7ahzf78/Xx/ME/r6v9c+3rGnxRk7e8uvIxY8YMbdq0SR9++KHCwsLc93iEh4crNDRUDodDc+fO1aJFixQbG6vY2FgtWrRInTp10jPPPOOzogG0Ly6Xyz2e1KmoqFBQUJC6d+/e4DYtv2raX9J9kpr/sUv9fTQ239L9t4Qv9uHN/nx9PH/w5evamufa1zX4oqZb41X4WLlypSQpOTnZY/natWs1depUSVJaWpouXryo6dOn6/Tp0xo+fLhycnIUFhbmk4IBtD+JiYn64x//6LEsJydHQ4cObfB+DwCBzavwUXcZtCkOh0MZGRnKyMhobk0AAty5c+f0zTffuOeLi4t16NAhdevWTdHR0UpPT9epU6e0YcMGSdK0adO0YsUKpaam6te//rX27t2rNWvW6L333mutLgDwo2Y/7QIAjTlw4IAeeugh93zdE21TpkzRunXrVFZWppKSEvf6mJgY7dixQy+99JLefPNNRUVFafny5TxmC7RThA8APpecnNzkldJ169bVW5aUlKSDBw/6sSoAbQW/agsAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrvA4fu3fv1oQJExQVFSWHw6EPPvjAY70xRhkZGYqKilJoaKiSk5N19OhRX9ULAAACnNfh4/z58xo8eLBWrFjR4PolS5Zo2bJlWrFihfbv3y+Xy6UxY8bo7NmzLS4WQOB46623FBMTo5CQECUkJOiTTz5ptG1eXp4cDke96auvvrJYMQBbgrzdYNy4cRo3blyD64wxysrK0oIFCzRx4kRJ0vr16xUREaFNmzbpxRdfbFm1AALC+++/r7lz5+qtt97S/fffr9///vcaN26cjh07pujo6Ea3KyoqUpcuXdzzd955p41yAVjm03s+iouLVV5erpSUFPcyp9OppKQk7dmzx5eHAtCGLVu2TM8//7xeeOEF9e/fX1lZWerdu7dWrlzZ5HY9e/aUy+VyT7fddpuligHY5NPwUV5eLkmKiIjwWB4REeFed6OamhpVV1d7TAACV21trQoKCjz+CJGklJSUm/4RMmTIEEVGRmr06NHatWuXP8sE0Ir88rSLw+HwmDfG1FtWJzMzU+Hh4e6pd+/e/igJgCWVlZW6evWqV3+EREZGavXq1crOztaWLVsUFxen0aNHa/fu3Y0ehz9cgMDl9T0fTXG5XJJ+vAISGRnpXl5RUVFvIKqTnp6u1NRU93x1dTUBBGgHvPkjJC4uTnFxce75xMRElZaWaunSpXrwwQcb3CYzM1MLFy70XcEArPHplY+YmBi5XC7l5ua6l9XW1io/P18jR45scBun06kuXbp4TAACV48ePXTbbbfVu8rR1B8hDRkxYoROnDjR6Pr09HRVVVW5p9LS0mbXDMAur698nDt3Tt988417vri4WIcOHVK3bt0UHR2tuXPnatGiRYqNjVVsbKwWLVqkTp066ZlnnvFp4QDapo4dOyohIUG5ubn6p3/6J/fy3NxcPf7447e8n8LCQo8rqDdyOp1yOp0tqhVA6/A6fBw4cEAPPfSQe77uI5MpU6Zo3bp1SktL08WLFzV9+nSdPn1aw4cPV05OjsLCwnxXNYA2LTU1VZMnT9bQoUOVmJio1atXq6SkRNOmTZP041WLU6dOacOGDZKkrKws9e3bVwMHDlRtba02btyo7OxsZWdnt2Y3APiJ1+EjOTlZxphG1zscDmVkZCgjI6MldQEIYE899ZR++OEHvfrqqyorK1N8fLx27NihPn36SJLKyspUUlLibl9bW6t58+bp1KlTCg0N1cCBA7V9+3aNHz++tboAwI98esMpANSZPn26pk+f3uC6devWecynpaUpLS3NQlUA2gJ+WA4AAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABgFeEDAABYRfgAAABWtcuvVz9+/Lj7v3v06KHo6OhWrAYAAFyvnYWPMkkd9Oyzz7qXhIR0UlHRcQIIAABtRDv72OWMpGuSNkoqkLRRly5dUGVlZatWBQAA/r92duWjTn9J97V2EQAAoAHt7MoHAABo6wgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCqnX7Phye+bh0AgLajnYcPvm4dAIC2pp1/7HJGfN06AABtSzu/8lGHr1sHAKCtaOdXPgAAQFtD+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABg1U/kS8Y88VsvAAC0np9Y+OC3XgAAaG0/sY9dzojfegEAoHX9xK581OG3XgAAaC0/sSsfAACgtRE+AACAVT/Rj13at5KSknr3sfBUT303vk68RgBgB+GjnSkpKVFcXH9dunTBYzlP9Xhq6HXiNQIAO/jYpZ2prKz82/9Q657o4amehtR/nXiNAMAWrny0WzzRc2t4nQDANq58AAAAqwgfAADAKj52uUFznoC42TY3rq+pqZHT6Wx0/mbb++KpDG/3ebM+eFuTL17nlryO1/++T0t4e+5b44matlADAFyP8HGd5jwBcbNtGn765DZJVxudv9n2LX0qw9t93kofvKnJV69zS1/HlmrOubf9RE1bqAEAbsTHLtdpzhMQN9um/vrf6sf/QTY2f7PtW/5Uhrf7vHkfvKvJN69zS1/H395SrS3pR1t4oqYt1AAAN/Jb+HjrrbcUExOjkJAQJSQk6JNPPvHXofyg7gmI/j7cpm59zE3mb7a9NzXdjLf79LZmXx+/qRq8fR1jGlnfHK1x7rxlvwZvx4D8/HwlJCQoJCRE/fr106pVqyxVCsA2v4SP999/X3PnztWCBQtUWFioUaNGady4cSopKfHH4QC0Md6OAcXFxRo/frxGjRqlwsJCzZ8/X7Nnz1Z2drblygHY4JfwsWzZMj3//PN64YUX1L9/f2VlZal3795auXKlPw4HoI3xdgxYtWqVoqOjlZWVpf79++uFF17Qr371Ky1dutRy5QBs8PkNp7W1tSooKNDLL7/ssTwlJUV79uyp176mpkY1NTXu+aqqKklSdXX1TY917ty5v/1XgaRzko57OV/041xBgc6dO6eioqIm10tShw4ddO3aNXcNN9um/nr/1lS/vfywz6Zrbuh1anp/9bf3vgZvX8eW9eFW+tGcf0++nm+shnPnzjX5HqtbZ4xptE1jvB0DJGnv3r1KSUnxWDZ27FitWbNGly9fVnBwcL1tmjt2eD9u6BbaePfvUWr5e+TG7X2xj5bU1Jz3te15X9TYGuPhzd/XLR/PbqUmn44bxsdOnTplJJnPPvvMY/lrr71m7r777nrtX3nlFSOJiYmpDU6lpaV+HwOMMSY2Nta89tprHss+++wzI8l89913DW7D2MHE1DanWxk3/PaorcPh8Jg3xtRbJknp6elKTU11z1+7dk3/93//p+7duzfY/nrV1dXq3bu3SktL1aVLF98Ublmg9yHQ65foQ0OMMTp79qyioqKavY9bHQOaat/Q8jrNGTvaw7luSHvtl0TfAok344bPw0ePHj102223qby83GN5RUWFIiIi6rV3Op0eXwwlSXfccYdXx+zSpUvAn7hA70Og1y/RhxuFh4c3aztvxwBJcrlcDbYPCgpS9+7dG9ymJWNHezjXDWmv/ZLoW6C41XHD5zecduzYUQkJCcrNzfVYnpubq5EjR/r6cADamOaMAYmJifXa5+TkaOjQoQ3e7wEgsPnlaZfU1FS98847evfdd3X8+HG99NJLKikp0bRp0/xxOABtzM3GgPT0dD333HPu9tOmTdPJkyeVmpqq48eP691339WaNWs0b9681uoCAD/yyz0fTz31lH744Qe9+uqrKisrU3x8vHbs2KE+ffr49DhOp1OvvPJKvUuvgSTQ+xDo9Uv0wR9uNgaUlZV5fOdHTEyMduzYoZdeeklvvvmmoqKitHz5cj355JM+rautvU6+0l77JdG39sphTDOepQMAAGgmftsFAABYRfgAAABWET4AAIBVhA8AAGBVwIYPb3+u21d2796tCRMmKCoqSg6HQx988IHHemOMMjIyFBUVpdDQUCUnJ+vo0aMebWpqajRr1iz16NFDnTt31j/+4z/qr3/9q0eb06dPa/LkyQoPD1d4eLgmT56sM2fOeLQpKSnRhAkT1LlzZ/Xo0UOzZ89WbW1tk/VnZmZq2LBhCgsLU8+ePfXEE09c9z3+gdGHlStXatCgQe4v5klMTNRHH30UMPXfKDMzUw6HQ3Pnzg3YPrQWb8eB/Px8JSQkKCQkRP369dOqVavqtcnOztaAAQPkdDo1YMAAbd261V/lN8mbvm3ZskVjxozRnXfe6X5PfPzxxx5t1q1bJ4fDUW+6dOmSv7tSjzd9y8vLa7Dur776yqNdWzhv3vRr6tSpDfZr4MCB7jZt6Zz53E2/gL0N2rx5swkODjZvv/22OXbsmJkzZ47p3LmzOXnypN+PvWPHDrNgwQKTnZ1tJJmtW7d6rF+8eLEJCwsz2dnZ5vDhw+app54ykZGRprq62t1m2rRp5u/+7u9Mbm6uOXjwoHnooYfM4MGDzZUrV9xtfv7zn5v4+HizZ88es2fPHhMfH28ee+wx9/orV66Y+Ph489BDD5mDBw+a3NxcExUVZWbOnNlk/WPHjjVr1641R44cMYcOHTKPPvqoiY6ONufOnQuYPmzbts1s377dFBUVmaKiIjN//nwTHBxsjhw5EhD1X++LL74wffv2NYMGDTJz5swJmHPQFng7Dnz77bemU6dOZs6cOebYsWPm7bffNsHBwea///u/3W327NljbrvtNrNo0SJz/Phxs2jRIhMUFGT27dtnq1vGGO/7NmfOHPP666+bL774wnz99dcmPT3dBAcHm4MHD7rbrF271nTp0sWUlZV5TLZ527ddu3YZSaaoqMij7uv/nbeF8+Ztv86cOePRn9LSUtOtWzfzyiuvuNu0lXPmDwEZPv7hH/7BTJs2zWPZPffcY15++WWrddwYPq5du2ZcLpdZvHixe9mlS5dMeHi4WbVqlTHmx39wwcHBZvPmze42p06dMh06dDA7d+40xhhz7NgxI8njjbN3714jyXz11VfGmB9DUIcOHcypU6fcbd577z3jdDpNVVXVLfehoqLCSDL5+fkB2wdjjOnatat55513Aqr+s2fPmtjYWJObm2uSkpLc4SOQ+tCavB0H0tLSzD333OOx7MUXXzQjRoxwz//iF78wP//5zz3ajB071vzyl7/0UdW3xhdj3IABA8zChQvd82vXrjXh4eG+KrHZvO1bXfg4ffp0o/tsC+etpeds69atxuFwmL/85S/uZW3lnPlDwH3sUvdz3Tf+/HZTP9dtS3FxscrLyz1qczqdSkpKctdWUFCgy5cve7SJiopSfHy8u83evXsVHh6u4cOHu9uMGDFC4eHhHm3i4+M9fsBn7NixqqmpUUFBwS3XXPcz5N26dQvIPly9elWbN2/W+fPnlZiYGFD1z5gxQ48++qgeeeQRj+WB1IfW0pxxYO/evfXajx07VgcOHNDly5ebbGNzbPHFGHft2jWdPXvW/b6uc+7cOfXp00e9evXSY489psLCQp/VfSta0rchQ4YoMjJSo0eP1q5duzzWtfZ588U5W7NmjR555JF6X8bZ2ufMXwIufFRWVurq1av1fqAqIiKi3g9T2VZ3/KZqKy8vV8eOHdW1a9cm2/Ts2bPe/nv27OnR5sbjdO3aVR07drzl18EYo9TUVD3wwAOKj48PqD4cPnxYt99+u5xOp6ZNm6atW7dqwIABAVP/5s2bdfDgQWVmZtZbFyh9aE3NGQca6mtERISuXLmiysrKJtvYfC18Mcb97ne/0/nz5/WLX/zCveyee+7RunXrtG3bNr333nsKCQnR/fffrxMnTvi0/qY0p2+RkZFavXq1srOztWXLFsXFxWn06NHavXu3u01rn7eWnrOysjJ99NFHeuGFFzyWt4Vz5i9++Xp1G7z9uW6bmlPbjW0aat+cNk2ZOXOmvvzyS3366af11rX1PsTFxenQoUM6c+aMsrOzNWXKFOXn5wdE/aWlpZozZ45ycnIUEhLSaLu23Ie2wtvXqKH2Ny5vK2NLc+t47733lJGRoQ8//NAjfI4YMUIjRoxwz99///2677779O///u9avny57wq/Bd70LS4uTnFxce75xMRElZaWaunSpXrwwQebtU9/aW4N69at0x133KEnnnjCY3lbOme+FnBXPprzc922uFwuSWqyNpfLpdraWp0+fbrJNt9//329/f/v//6vR5sbj3P69Gldvnz5ll6HWbNmadu2bdq1a5d69eoVcH3o2LGj7rrrLg0dOlSZmZkaPHiw3njjjYCov6CgQBUVFUpISFBQUJCCgoKUn5+v5cuXKygoyL1tW+5Da2vOONBQXysqKhQUFKTu3bs32cbma9GSMe7999/X888/r//6r/+q93HejTp06KBhw4ZZ/SvaV+P3iBEjPOpu7fPWkn4ZY/Tuu+9q8uTJ6tixY5NtW+Oc+UvAhY/m/Fy3LTExMXK5XB611dbWKj8/311bQkKCgoODPdqUlZXpyJEj7jaJiYmqqqrSF1984W7z+eefq6qqyqPNkSNHVFZW5m6Tk5Mjp9OphISERms0xmjmzJnasmWL/vznPysmJibg+tBYv2pqagKi/tGjR+vw4cM6dOiQexo6dKgmTZqkQ4cOqV+/fm2+D62tOeNAYmJivfY5OTkaOnSogoODm2xjc2xp7hj33nvvaerUqdq0aZMeffTRmx7HGKNDhw4pMjKyxTXfKl+N34WFhR51t/Z5a0m/8vPz9c033+j555+/6XFa45z5jY27Wn2t7pGmNWvWmGPHjpm5c+eazp07e9wl7C9nz541hYWFprCw0Egyy5YtM4WFhe7HqRYvXmzCw8PNli1bzOHDh83TTz/d4COSvXr1Mn/605/MwYMHzcMPP9zgI5KDBg0ye/fuNXv37jX33ntvg49Ijh492hw8eND86U9/Mr169brpI5K/+c1vTHh4uMnLy/N4dOvChQvuNm29D+np6Wb37t2muLjYfPnll2b+/PmmQ4cOJicnJyDqb8j1T7sEah9su9k48PLLL5vJkye729c9avvSSy+ZY8eOmTVr1tR71Pazzz4zt912m1m8eLE5fvy4Wbx4cas+anurfdu0aZMJCgoyb775psf7+syZM+42GRkZZufOneZ//ud/TGFhofnXf/1XExQUZD7//PM23bd/+7d/M1u3bjVff/21OXLkiHn55ZeNJJOdne1u0xbOm7f9qvPss8+a4cOHN7jPtnLO/CEgw4cxxrz55pumT58+pmPHjua+++5zPyrqb3WPfd04TZkyxRjz42OSr7zyinG5XMbpdJoHH3zQHD582GMfFy9eNDNnzjTdunUzoaGh5rHHHjMlJSUebX744QczadIkExYWZsLCwsykSZPqPWp28uRJ8+ijj5rQ0FDTrVs3M3PmTHPp0qUm62+odklm7dq17jZtvQ+/+tWv3Of+zjvvNKNHj3YHj0CovyE3ho9A7ENraGocmDJliklKSvJon5eXZ4YMGWI6duxo+vbta1auXFlvn3/4wx9MXFycCQ4ONvfcc4/H/+Rs8qZvSUlJTY5Lxhgzd+5cEx0d7X7fpKSkmD179ljs0f/nTd9ef/118/d///cmJCTEdO3a1TzwwANm+/bt9fbZFs6bt/8ez5w5Y0JDQ83q1asb3F9bOme+5jDmb3dcAQAAWBBw93wAAIDARvgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABg1f8DfLxSTX485D4AAAAASUVORK5CYII=", + "text/plain": [ + "
    " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "ax = plt.subplot(1, 2, 1)\n", + "plt.hist(weight, color='blue', edgecolor='black', bins=50)\n", + "ax = plt.subplot(1, 2, 2)\n", + "plt.hist(weight[weight <= 1], color='blue', edgecolor='black', bins=50)\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 75, "id": "e6a61e8d-5c55-4312-a48a-a168817e7d77", "metadata": {}, "outputs": [ @@ -878,7 +1113,7 @@ "Name: raw_specifications, Length: 167, dtype: object" ] }, - "execution_count": 404, + "execution_count": 75, "metadata": {}, "output_type": "execute_result" } @@ -949,7 +1184,7 @@ }, { "cell_type": "code", - "execution_count": 419, + "execution_count": 21, "id": "f6fcaae9-9c24-450c-8737-8b6aa0e406a9", "metadata": {}, "outputs": [ @@ -970,7 +1205,7 @@ "Name: raw_specifications, Length: 167, dtype: object" ] }, - "execution_count": 419, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -983,10 +1218,21 @@ }, { "cell_type": "code", - "execution_count": 429, + "execution_count": 41, "id": "f6ed2736-26d2-403f-9ab0-a228057176e4", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{NoneType, dict}" + ] + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "def clean_origin_name(origin: str) -> str:\n", " mapping = {\n", @@ -999,27 +1245,140 @@ " return mapping[origin]\n", "\n", "origin = data[\"raw_specifications\"].apply(parse_value, value=\"origin\")\n", - "clean_origin = origin.apply(clean_origin_name)\n" + "clean_origin = origin.apply(clean_origin_name)\n", + "clean_origin" ] }, { "cell_type": "code", - "execution_count": 453, + "execution_count": 46, "id": "d2df0c7e-cd03-4628-ae29-fae4f0f37dfa", "metadata": {}, "outputs": [ { "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    materialspackagingoriginweightheightwidthdepthtcinprimary_category
    4[metal]1imported2109.20280058.4202.54058.42086345566Home
    5[glass]1mixed0.08029720.3204.4454.44575552641Holiday Shop
    6[wood]1mixed0.27530313.33510.16010.16081632725Holiday Shop
    11[fabric]1mixed2.72680733.02020.32020.32081726538Holiday Shop
    15[metal]1imported1360.77600074.9301.27031.11586435324Home
    \n", + "
    " + ], "text/plain": [ - "(57, 9)" + " materials packaging origin weight height width depth \\\n", + "4 [metal] 1 imported 2109.202800 58.420 2.540 58.420 \n", + "5 [glass] 1 mixed 0.080297 20.320 4.445 4.445 \n", + "6 [wood] 1 mixed 0.275303 13.335 10.160 10.160 \n", + "11 [fabric] 1 mixed 2.726807 33.020 20.320 20.320 \n", + "15 [metal] 1 imported 1360.776000 74.930 1.270 31.115 \n", + "\n", + " tcin primary_category \n", + "4 86345566 Home \n", + "5 75552641 Holiday Shop \n", + "6 81632725 Holiday Shop \n", + "11 81726538 Holiday Shop \n", + "15 86435324 Home " ] }, - "execution_count": 453, + "execution_count": 46, "metadata": {}, "output_type": "execute_result" } ], "source": [ + "def get_val(xs, i):\n", + " if xs is None:\n", + " return None\n", + " return xs.get(i)\n", + "\n", "clean_data = pd.DataFrame(\n", " data={\n", " \"materials\": clean_materials,\n", @@ -1036,7 +1395,31 @@ "\n", "clean_data\n", "\n", - "clean_data[~clean_data.isnull().any(axis=1)].shape" + "clean_data[~clean_data.isnull().any(axis=1)].head()" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "427c5d6a-8f8b-4441-89c5-193d647b49e2", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "4 23271231140\n", + "Name: gtin13, dtype: int64" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "weight = data[\"raw_specifications\"].apply(parse_value, value=\"weight\")\n", + "sel = weight == \" 4.65 pounds\"\n", + "data.loc[sel, \"gtin13\"]" ] } ],