{ "cells": [ { "cell_type": "code", "execution_count": 47, "id": "98ded03d-0208-4416-a5e5-720a2e0742fa", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
titleurlbrandmain_imageskudescriptionraw_descriptiongtin13currencyprice...sub_category_1sub_category_2sub_category_3imagesraw_specificationsspecificationshighlightsraw_highlightsuniq_idscraped_at
0NMR Distribution Zoltar Fortunes Playing Cards...https://www.target.com/p/nmr-distribution-zolt...NMR Distributionhttps://target.scene7.com/is/image/Target/GUES...81917300Zoltar the Great Gypsy can see your future… Yo...<div class=\"h-margin-v-default\" data-test=\"ite...840391145528USD10.99...GamesAdult GamesNaNhttps://target.scene7.com/is/image/Target/GUES...<div class=\"styles__StyledCol-sc-ct8kx6-0 iKGd...Suggested Age: 6 Years and Up | Suggested Age:...ZOLTAR CARDS: Zoltar the Great Gypsy can see y...<li class=\"styles__Bullet-sc-6aebpn-0 eIfLaI\">...898ca3c8-8bfa-5fac-a48e-53879845cf4806/12/22
1The Gifted Stationery 2021 - 2022 Monthly Wall...https://www.target.com/p/the-gifted-stationery...The Gifted Stationaryhttps://target.scene7.com/is/image/Target/GUES...8482100716 month wall calendar provides easy planning ...<div class=\"h-margin-v-default\" data-test=\"ite...9781801433983USD12.99...CalendarsNaNNaNhttps://target.scene7.com/is/image/Target/GUES...<div class=\"styles__StyledCol-sc-ct8kx6-0 iKGd...Dimensions (Overall): 11.9 inches (L), 11.9 in...16-MONTH CALENDAR: Easy planning and goal sett...<li class=\"styles__Bullet-sc-6aebpn-0 eIfLaI\">...b7c8e8ce-55b6-529d-b52e-dbfbd70b066e06/12/22
2Doctor Who: Series Three, Part Two (DVD)https://www.target.com/p/doctor-who-series-thr...Warner Bros.https://target.scene7.com/is/image/Target/GUES...15432753Tenth Doctor David Tennant (Broadchurch, Harry...<div class=\"h-margin-v-default\" data-test=\"ite...883929408115USD10.89...MoviesNaNNaNhttps://target.scene7.com/is/image/Target/GUES...<div class=\"styles__StyledCol-sc-ct8kx6-0 iKGd...Movie Category: Television | Movie Category: T...Run Time: 315:00 min | Disc Count: 2 | Rating:...<li class=\"styles__Bullet-sc-6aebpn-0 eIfLaI\">...97fd83d8-e76b-5bac-999c-b734fdcafe6606/12/22
3Blue Panda 75 Pieces Tie Dye Birthday Party Su...https://www.target.com/p/blue-panda-75-pieces-...Blue Pandahttps://target.scene7.com/is/image/Target/GUES...84199597Throwing a party has never been easier, we’ve ...<div class=\"h-margin-v-default\" data-test=\"ite...194425194489USD26.99...Birthday Party SuppliesNaNNaNhttps://target.scene7.com/is/image/Target/GUES...<div class=\"styles__StyledCol-sc-ct8kx6-0 iKGd...Number of Pieces: 24 | Number of Pieces: 24 | ...Serves 24: Includes 1 plastic table cover, a b...<li class=\"styles__Bullet-sc-6aebpn-0 eIfLaI\">...3b4d60b0-0888-596e-babe-d9e6e4e3131c06/12/22
4Sullivans Hammered Metal Wall Medallions Set o...https://www.target.com/p/sullivans-hammered-me...Sullivanshttps://target.scene7.com/is/image/Target/GUES...86345566Bring a little charm and delight to your space...<div class=\"h-margin-v-default\" data-test=\"ite...23271231140USD118.99...Home DecorWall DecorWall Accentshttps://target.scene7.com/is/image/Target/GUES...<div class=\"styles__StyledCol-sc-ct8kx6-0 iKGd...Dimensions (Overall): 23 inches (H) x 1 inches...This unique wall set will be the perfect addit...<li class=\"styles__Bullet-sc-6aebpn-0 eIfLaI\">...036188d9-4546-5952-af8f-bc234d1f811306/12/22
\n", "

5 rows × 24 columns

\n", "
" ], "text/plain": [ " title \\\n", "0 NMR Distribution Zoltar Fortunes Playing Cards... \n", "1 The Gifted Stationery 2021 - 2022 Monthly Wall... \n", "2 Doctor Who: Series Three, Part Two (DVD) \n", "3 Blue Panda 75 Pieces Tie Dye Birthday Party Su... \n", "4 Sullivans Hammered Metal Wall Medallions Set o... \n", "\n", " url brand \\\n", "0 https://www.target.com/p/nmr-distribution-zolt... NMR Distribution \n", "1 https://www.target.com/p/the-gifted-stationery... The Gifted Stationary \n", "2 https://www.target.com/p/doctor-who-series-thr... Warner Bros. \n", "3 https://www.target.com/p/blue-panda-75-pieces-... Blue Panda \n", "4 https://www.target.com/p/sullivans-hammered-me... Sullivans \n", "\n", " main_image sku \\\n", "0 https://target.scene7.com/is/image/Target/GUES... 81917300 \n", "1 https://target.scene7.com/is/image/Target/GUES... 84821007 \n", "2 https://target.scene7.com/is/image/Target/GUES... 15432753 \n", "3 https://target.scene7.com/is/image/Target/GUES... 84199597 \n", "4 https://target.scene7.com/is/image/Target/GUES... 86345566 \n", "\n", " description \\\n", "0 Zoltar the Great Gypsy can see your future… Yo... \n", "1 16 month wall calendar provides easy planning ... \n", "2 Tenth Doctor David Tennant (Broadchurch, Harry... \n", "3 Throwing a party has never been easier, we’ve ... \n", "4 Bring a little charm and delight to your space... \n", "\n", " raw_description gtin13 currency \\\n", "0
... \n", "1
  • ... \n", "2
  • ... \n", "3
  • ... \n", "4
  • ... \n", "\n", " uniq_id scraped_at \n", "0 898ca3c8-8bfa-5fac-a48e-53879845cf48 06/12/22 \n", "1 b7c8e8ce-55b6-529d-b52e-dbfbd70b066e 06/12/22 \n", "2 97fd83d8-e76b-5bac-999c-b734fdcafe66 06/12/22 \n", "3 3b4d60b0-0888-596e-babe-d9e6e4e3131c 06/12/22 \n", "4 036188d9-4546-5952-af8f-bc234d1f8113 06/12/22 \n", "\n", "[5 rows x 24 columns]" ] }, "execution_count": 47, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import functools\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "%matplotlib inline\n", "from IPython.display import display, HTML\n", "\n", "in_file = \"/home/jovyan/data/large_target_store_products_dataset_sample - large_target_store_products_dataset_sample.csv\"\n", "data = pd.read_csv(in_file)\n", "data.head()" ] }, { "cell_type": "code", "execution_count": 9, "id": "c20c14ea-9ef6-4d40-8b7d-4731d0866239", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Index(['title', 'url', 'brand', 'main_image', 'sku', 'description',\n", " 'raw_description', 'gtin13', 'currency', 'price', 'availability',\n", " 'availableDeliveryMethod', 'available_branch', 'primary_category',\n", " 'sub_category_1', 'sub_category_2', 'sub_category_3', 'images',\n", " 'raw_specifications', 'specifications', 'highlights', 'raw_highlights',\n", " 'uniq_id', 'scraped_at'],\n", " dtype='object')" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "data.axes[1]" ] }, { "cell_type": "code", "execution_count": 3, "id": "165723d1-8152-4e30-b25a-cbca7f4935f9", "metadata": {}, "outputs": [ { "data": { "text/html": [ "

    Specifications

    Suggested Age: 6 Years and Up
    CPSC Choking Hazard Warnings: Choking_hazard_small_parts
    TCIN: 81917300
    UPC: 840391145528
    Origin: imported

    The above item details were provided by the Target Plus™ Partner. Target does not represent or warrant that this information is accurate or complete. On occasion, manufacturers may modify their items and update their labels.

    We recommend that you do not rely solely on the information presented. If you have a specific question about this item, you may consult the item's label, contact the manufacturer directly or call Target Guest Services at 1-800-591-3869.

    " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "def render_html(html: str):\n", " \"\"\"Render an html string\"\"\"\n", " display(HTML(html))\n", "\n", "render_html(data.loc[0, 'raw_specifications'])" ] }, { "cell_type": "code", "execution_count": 5, "id": "d153a732-f18c-4fb7-9282-9c02c49c3cc0", "metadata": {}, "outputs": [ { "data": { "text/html": [ "

    Specifications

    Dimensions (Overall): 11.9 inches (L), 11.9 inches (W)
    Dated format: Monthly
    Calendar year: 2022
    Material: Paper
    TCIN: 84821007
    UPC: 9781801433983
    Origin: imported

    The above item details were provided by the Target Plus™ Partner. Target does not represent or warrant that this information is accurate or complete. On occasion, manufacturers may modify their items and update their labels.

    We recommend that you do not rely solely on the information presented. If you have a specific question about this item, you may consult the item's label, contact the manufacturer directly or call Target Guest Services at 1-800-591-3869.

    " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "def look_for_matches(data: pd.DataFrame, pattern: str, colname : str = \"raw_specifications\") -> str:\n", " \"\"\"Useful for finding cells in raw_specifications containing a given string\"\"\"\n", " return data.loc[data.loc[:, colname].str.contains(pattern), colname].iloc[0]\n", "\n", "render_html(look_for_matches(data, \"Material\"))" ] }, { "cell_type": "code", "execution_count": 6, "id": "b06e66af-e0e3-466c-8797-546d5e076f32", "metadata": {}, "outputs": [ { "data": { "text/html": [ "

    Specifications

    Contains: Does Not Contain Any of the 8 Major Allergens
    Dietary Needs: Gluten Free
    Form: Pieces
    State of Readiness: Ready to Eat
    Package Quantity: 1
    Net weight: 15.6 Ounces
    TCIN: 54571204
    UPC: 022000279729
    Item Number (DPCI): 055-02-1211
    Origin: Made in the USA or Imported
    Grocery Disclaimer:
    Content on this site is for reference purposes only. Target does not represent or warrant that the nutrition, ingredient, allergen and other product information on our Web or Mobile sites are accurate or complete, since this information comes from the product manufacturers. On occasion, manufacturers may improve or change their product formulas and update their labels. We recommend that you do not rely solely on the information presented on our Web or Mobile sites and that you review the product's label or contact the manufacturer directly if you have specific product concerns or questions. If you have specific healthcare concerns or questions about the products displayed, please contact your licensed healthcare professional for advice or answers. Any additional pictures are suggested servings only.
    " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "render_html(look_for_matches(data, \"Package\"))" ] }, { "cell_type": "code", "execution_count": 7, "id": "015a0d99-b4c2-47cf-9e23-59485a450470", "metadata": {}, "outputs": [ { "data": { "text/html": [ "

    Specifications

    Dimensions (Overall): 23 inches (H) x 1 inches (W) x 23 inches (D)
    Weight: 4.65 pounds
    Art subject: Geometric Shapes
    Orientation: Vertical
    Material: Metal
    Battery: No Battery Used
    TCIN: 86345566
    UPC: 023271231140
    Origin: imported

    The above item details were provided by the Target Plus™ Partner. Target does not represent or warrant that this information is accurate or complete. On occasion, manufacturers may modify their items and update their labels.

    We recommend that you do not rely solely on the information presented. If you have a specific question about this item, you may consult the item's label, contact the manufacturer directly or call Target Guest Services at 1-800-591-3869.

    " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "render_html(look_for_matches(data, \"Weight\"))" ] }, { "cell_type": "code", "execution_count": 8, "id": "99cbfba8-4324-4d2a-b2e2-d766a5e57964", "metadata": {}, "outputs": [ { "data": { "text/html": [ "

    Specifications

    Dimensions (Overall): 11.9 inches (L), 11.9 inches (W)
    Dated format: Monthly
    Calendar year: 2022
    Material: Paper
    TCIN: 84821007
    UPC: 9781801433983
    Origin: imported

    The above item details were provided by the Target Plus™ Partner. Target does not represent or warrant that this information is accurate or complete. On occasion, manufacturers may modify their items and update their labels.

    We recommend that you do not rely solely on the information presented. If you have a specific question about this item, you may consult the item's label, contact the manufacturer directly or call Target Guest Services at 1-800-591-3869.

    " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "render_html(look_for_matches(data, \"Dimensions\"))" ] }, { "cell_type": "code", "execution_count": 9, "id": "6c05e1db-7181-422e-924b-e67a28886abe", "metadata": {}, "outputs": [ { "data": { "text/html": [ "

    Specifications

    Suggested Age: 6 Years and Up
    CPSC Choking Hazard Warnings: Choking_hazard_small_parts
    TCIN: 81917300
    UPC: 840391145528
    Origin: imported

    The above item details were provided by the Target Plus™ Partner. Target does not represent or warrant that this information is accurate or complete. On occasion, manufacturers may modify their items and update their labels.

    We recommend that you do not rely solely on the information presented. If you have a specific question about this item, you may consult the item's label, contact the manufacturer directly or call Target Guest Services at 1-800-591-3869.

    " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "render_html(look_for_matches(data, \"TCIN\"))" ] }, { "cell_type": "code", "execution_count": 10, "id": "bf3a07d9-11c0-48c4-be03-373a3ba5f755", "metadata": {}, "outputs": [ { "data": { "text/html": [ "

    Specifications

    Suggested Age: 6 Years and Up
    CPSC Choking Hazard Warnings: Choking_hazard_small_parts
    TCIN: 81917300
    UPC: 840391145528
    Origin: imported

    The above item details were provided by the Target Plus™ Partner. Target does not represent or warrant that this information is accurate or complete. On occasion, manufacturers may modify their items and update their labels.

    We recommend that you do not rely solely on the information presented. If you have a specific question about this item, you may consult the item's label, contact the manufacturer directly or call Target Guest Services at 1-800-591-3869.

    " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "render_html(look_for_matches(data, \"Origin\"))" ] }, { "cell_type": "code", "execution_count": 11, "id": "3853dd07-1977-44c6-bc97-2ecdaa98980e", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "raw_specifications\n", "False 124\n", "True 43\n", "Name: count, dtype: int64" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "colname = \"raw_specifications\"\n", "patterns = [\"Material\", \"Package Quantity\", \"Weight\", \"Dimensions\", \"TCIN\", \"Origin\"]\n", "\n", "(data.loc[:, colname].str.contains(\"Package Quantity\") | data.loc[:, colname].str.contains(\"Number of Pieces\")).value_counts()\n", "\n", "# Package Quantity and Number of Pieces are never found together. Maybe they refer to the same thing?" ] }, { "cell_type": "code", "execution_count": 12, "id": "be8a7628-9199-4f25-a533-d041a7ff540c", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
    \n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
    titleurlbrandmain_imageskudescriptionraw_descriptiongtin13currencyprice...sub_category_1sub_category_2sub_category_3imagesraw_specificationsspecificationshighlightsraw_highlightsuniq_idscraped_at
    8NCAA Illinois Fighting Illini Circo Cheese Cut...https://www.target.com/p/ncaa-illinois-fightin...NCAAhttps://target.scene7.com/is/image/Target/GUES...79646040Reach out to the complex cheese lover in your ...<div class=\"h-margin-v-default\" data-test=\"ite...99967205276USD58.95...Sports Fan ShopSports Fan Shop Home GoodsSports Fan Shop Barware & Drinkwarehttps://target.scene7.com/is/image/Target/GUES...<div class=\"styles__StyledCol-sc-ct8kx6-0 iKGd...Number of Pieces: 5 | Number of Pieces: 5 | We...BEAUTY & ELEGANCE - The Circo swivel-style cir...<li class=\"styles__Bullet-sc-6aebpn-0 eIfLaI\">...0c549116-75c8-56cb-8877-165380d0efd906/12/22
    13Blue Panda Jumbo Dinosaur Floor Puzzle, Double...https://www.target.com/p/blue-panda-jumbo-dino...Blue Pandahttps://target.scene7.com/is/image/Target/GUES...80405355Package Includes\\r\\nLarge Dinosaur Floor Puzzl...<div class=\"h-margin-v-default\" data-test=\"ite...194425203808USD19.99...PuzzlesNaNNaNhttps://target.scene7.com/is/image/Target/GUES...<div class=\"styles__StyledCol-sc-ct8kx6-0 iKGd...Number of Pieces: 17 | Number of Pieces: 17 | ...JUMBO DINOSAUR PUZZLE: This t-rex foam puzzle ...<li class=\"styles__Bullet-sc-6aebpn-0 eIfLaI\">...151c72b4-4856-502f-a508-961cc81fffa906/12/22
    14Women's Round Aviator Sunglasses - Universal T...https://www.target.com/p/women-39-s-round-avia...Universal Threadhttps://target.scene7.com/is/image/Target/GUES...84201225Round out your eyewear collection with the Rou...<div class=\"h-margin-v-default\" data-test=\"ite...195995526496USD15.00...Eye CareNaNNaNhttps://target.scene7.com/is/image/Target/GUES...<div class=\"styles__StyledCol-sc-ct8kx6-0 iKGd...Material: Metal (Frame) | Material: Metal (Fra...Universal Thread round aviator sunglasses with...<li class=\"styles__Bullet-sc-6aebpn-0 eIfLaI\">...2a803c0f-00bf-50a6-a490-d381620ac3a306/12/22
    \n", "

    3 rows × 24 columns

    \n", "
    " ], "text/plain": [ " title \\\n", "8 NCAA Illinois Fighting Illini Circo Cheese Cut... \n", "13 Blue Panda Jumbo Dinosaur Floor Puzzle, Double... \n", "14 Women's Round Aviator Sunglasses - Universal T... \n", "\n", " url brand \\\n", "8 https://www.target.com/p/ncaa-illinois-fightin... NCAA \n", "13 https://www.target.com/p/blue-panda-jumbo-dino... Blue Panda \n", "14 https://www.target.com/p/women-39-s-round-avia... Universal Thread \n", "\n", " main_image sku \\\n", "8 https://target.scene7.com/is/image/Target/GUES... 79646040 \n", "13 https://target.scene7.com/is/image/Target/GUES... 80405355 \n", "14 https://target.scene7.com/is/image/Target/GUES... 84201225 \n", "\n", " description \\\n", "8 Reach out to the complex cheese lover in your ... \n", "13 Package Includes\\r\\nLarge Dinosaur Floor Puzzl... \n", "14 Round out your eyewear collection with the Rou... \n", "\n", " raw_description gtin13 currency \\\n", "8
    ... \n", "13
  • ... \n", "14
  • ... \n", "\n", " uniq_id scraped_at \n", "8 0c549116-75c8-56cb-8877-165380d0efd9 06/12/22 \n", "13 151c72b4-4856-502f-a508-961cc81fffa9 06/12/22 \n", "14 2a803c0f-00bf-50a6-a490-d381620ac3a3 06/12/22 \n", "\n", "[3 rows x 24 columns]" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "sel = data.loc[:, \"raw_specifications\"].str.contains(\"Dimensions\")\n", "dimensions = data.loc[sel]\n", "sel_overall = ~dimensions.loc[sel, \"raw_specifications\"].str.contains(\"Overall\")\n", "#for x in dimensions.loc[sel_overall, \"raw_specifications\"]:\n", "# render_html(x)\n", "dimensions.loc[sel_overall]\n", "\n", "# looks like \"dimenions\" can be \"Dimensions (Overall)\", \"Dimensions\" or other things\n", "# like \"Assembled Dimensions\" or \"Piece X Dimensions\". But this latter two options are\n", "# incomplete (lack the height), harder to parse and rare enough that I'll just drop them" ] }, { "cell_type": "code", "execution_count": 13, "id": "3fd14c61-c1f8-4b18-b04a-9b616e6cf9a7", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "\"Dimensions (Overall): 11.9 inches (L), 11.9 inches (W) | Dimensions (Overall): 11.9 inches (L), 11.9 inches (W) | Dated format: Monthly | Dated format: Monthly | Calendar year: 2022 | Calendar year: 2022 | Material: Paper | Material: Paper | TCIN: 84821007 | UPC: 9781801433983 | Origin: imported | The above item details were provided by the Target Plus™ Partner. Target does not represent or warrant that this information is accurate or complete. On occasion, manufacturers may modify their items and update their labels.We recommend that you do not rely solely on the information presented. If you have a specific question about this item, you may consult the item's label, contact the manufacturer directly or call Target Guest Services at 1-800-591-3869.\"" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import xml.etree.ElementTree as ET\n", "\n", "# There seem to be dupliucates on \"specifications\" that are not \n", "# found on \"raw_specifications\"\n", "# I think it's safe to just remove the duplicates\n", "\n", "specifications = data.loc[1,\"specifications\"]\n", "specifications\n", "\n" ] }, { "cell_type": "code", "execution_count": 14, "id": "88894d7c-dc68-42ee-9c9d-a53474762a7a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0 None\n", "1 Paper\n", "2 None\n", "3 Paper\n", "4 Metal\n", " ... \n", "162 Plastic\n", "163 None\n", "164 Fabric\n", "165 None\n", "166 Stoneware\n", "Name: raw_specifications, Length: 167, dtype: object\n", "0 None\n", "1 None\n", "2 None\n", "3 24\n", "4 None\n", " ... \n", "162 None\n", "163 None\n", "164 None\n", "165 None\n", "166 1\n", "Name: raw_specifications, Length: 167, dtype: object\n", "0 None\n", "1 11.9 inches (L), 11.9 inches (W)\n", "2 None\n", "3 None\n", "4 23 inches (H) x 1 inches (W) x 23 inches (D)\n", " ... \n", "162 5.0 inches (W) x 9.5 inches (D)\n", "163 12.0 inches (H) x 12.0 inches (W)\n", "164 4.75 inches (H) x 2.25 inches (W) x 2.25 inch...\n", "165 None\n", "166 4.5 inches (H) x 12.5 inches (W) x 4.5 inches...\n", "Name: raw_specifications, Length: 167, dtype: object\n", "0 None\n", "1 None\n", "2 None\n", "3 None\n", "4 4.65 pounds\n", " ... \n", "162 .28 pounds\n", "163 32.0 pounds\n", "164 None\n", "165 None\n", "166 None\n", "Name: raw_specifications, Length: 167, dtype: object\n", "0 81917300\n", "1 84821007\n", "2 15432753\n", "3 84199597\n", "4 86345566\n", " ... \n", "162 83388852\n", "163 80836585\n", "164 75477923\n", "165 85634544\n", "166 80239765\n", "Name: raw_specifications, Length: 167, dtype: object\n", "0 imported\n", "1 imported\n", "2 Made in the USA\n", "3 imported\n", "4 imported\n", " ... \n", "162 made in the USA or imported\n", "163 made in the USA or imported\n", "164 made in the USA or imported\n", "165 imported\n", "166 imported\n", "Name: raw_specifications, Length: 167, dtype: object\n" ] } ], "source": [ "def parse_specs(specifications: str) -> dict[str,str]:\n", " fields_mapping = {\n", " \"Material\": \"materials\",\n", " \"Package Quantity\": \"packaging\",\n", " \"Number of Pieces\": \"packaging\",\n", " \"Dimensions (Overall)\": \"dimensions\",\n", " \"Dimensions\": \"dimensions\",\n", " \"Weight\": \"weight\",\n", " \"TCIN\": \"tcin\",\n", " \"Origin\": \"origin\", \n", " }\n", " spec_dict = {}\n", " for spec in specifications.split(\"|\"):\n", " if \":\" in spec:\n", " try:\n", " field, value = spec.split(\":\")\n", " except ValueError:\n", " print(spec)\n", " return {}\n", " field = field.strip()\n", " if field in fields_mapping:\n", " field = fields_mapping[field]\n", " spec_dict[field] = value.strip()\n", " return spec_dict\n", "\n", "\n", "\n", "def iter_parse(root: ET.Element) -> dict[str,str]:\n", " \"\"\"Recursively parse the XML tree into a dictionary\n", " Each key/value pair is inside it's own
    tag and\n", " the key inside a tag.\n", " The fields that I believe are compulsory (TCIN, UPC and Origin)\n", " are only nested one level deep, while the rest of fields seem\n", " to be always nested two levels deep. But parsing it recursively\n", " helps generalise both cases.\"\"\"\n", " \n", " spec_dict = {}\n", " for child in root:\n", " if child.tag == \"div\":\n", " if \"b\" in [x.tag for x in child]:\n", " key, *values = child.itertext()\n", " key = key.strip(\":\")\n", " value = \"\".join(values).strip(\":\")\n", " spec_dict[key] = value\n", " else:\n", " spec_dict.update(iter_parse(child))\n", " return spec_dict\n", "\n", "def parse_raw_specs(raw_specs: str) -> dict[str,str]:\n", " \"\"\"Parse a raw specifications XML string into a dictionary\n", " This involves first recursively parsing the XML tree and then\n", " renaming the key values\"\"\"\n", " \n", " fields_mapping = {\n", " \"Material\": \"materials\",\n", " \"Package Quantity\": \"packaging\",\n", " \"Number of Pieces\": \"packaging\",\n", " \"Dimensions (Overall)\": \"dimensions\",\n", " \"Dimensions\": \"dimensions\",\n", " \"Weight\": \"weight\",\n", " \"TCIN\": \"tcin\",\n", " \"Origin\": \"origin\", \n", " }\n", " xml_root = ET.fromstring(raw_specs)\n", " parsed = iter_parse(xml_root)\n", " specs_dict = {\n", " fields_mapping[key]: value\n", " for key, value in parsed.items()\n", " if key in fields_mapping\n", " }\n", " return specs_dict\n", " \n", "\n", "def parse_value(specs: str, value: str) -> str:\n", " return parse_raw_specs(specs).get(value)\n", "\n", "for x in [\"materials\", \"packaging\", \"dimensions\", \"weight\", \"tcin\", \"origin\"]:\n", " print(data[\"raw_specifications\"].apply(parse_value, value=x))\n" ] }, { "cell_type": "code", "execution_count": 17, "id": "5902f1b3-a77e-4353-92aa-71cc729e8c87", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0 None\n", "1 {'width': 30.226000000000003}\n", "2 None\n", "3 None\n", "4 {'height': 58.42, 'width': 2.54, 'depth': 58.42}\n", " ... \n", "162 {'width': 12.7, 'depth': 24.13}\n", "163 {'height': 30.48, 'width': 30.48}\n", "164 {'height': 12.065, 'width': 5.715, 'depth': 5....\n", "165 None\n", "166 {'height': 11.43, 'width': 31.75, 'depth': 11.43}\n", "Name: raw_specifications, Length: 167, dtype: object" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import re\n", "from typing import Optional\n", "\n", "def parse_dimensions_measure(dimensions: str, measure: str) -> Optional[dict[str,str]]:\n", " expr = f\"(?P\\d*[.,]?\\d*)\\s+(?P[a-zA-Z]*)\\s+\\({measure}\\)\" \n", " if match := re.search(expr, dimensions):\n", " return {\n", " \"value\": float(match.group(\"value\")),\n", " \"unit\": match.group(\"unit\").lower()\n", " }\n", "\n", "def units_to_cm(value: float, unit: str) -> float:\n", " conversions = {\n", " \"inches\": 2.54,\n", " \"feet\": 30.48,\n", " \"cm\": 1\n", " }\n", " return value * conversions[unit]\n", "\n", "\n", "def parse_dimensions(dimensions: Optional[str]) -> Optional[dict[str,float]]:\n", " if dimensions is None:\n", " return None\n", " height = parse_dimensions_measure(dimensions, \"H\")\n", " width = parse_dimensions_measure(dimensions, \"W\")\n", " depth = parse_dimensions_measure(dimensions, \"D\")\n", " dimensions = {\n", " \"height\": height,\n", " \"width\": width,\n", " \"depth\": depth,\n", " }\n", " return {\n", " key: units_to_cm(**value)\n", " for key,value in dimensions.items()\n", " if value is not None\n", " }\n", "\n", "dimensions = data[\"raw_specifications\"].apply(parse_value, value=\"dimensions\").apply(parse_dimensions)\n", "dimensions" ] }, { "cell_type": "code", "execution_count": 51, "id": "26ece1d2-c466-498c-aeb6-5610fb16c7d3", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0 NaN\n", "1 NaN\n", "2 NaN\n", "3 NaN\n", "4 2109.202800\n", " ... \n", "162 127.005760\n", "163 14514.944000\n", "164 0.078812\n", "165 NaN\n", "166 0.829595\n", "Name: raw_specifications, Length: 167, dtype: float64" ] }, "execution_count": 51, "metadata": {}, "output_type": "execute_result" } ], "source": [ "def units_to_g(value: float, unit: str) -> float:\n", " conversions = {\n", " \"pounds\": 453.592,\n", " \"ounces\": 28.3495,\n", " \"g\": 1\n", " }\n", " return value * conversions[unit]\n", "\n", "def parse_weight(weight: str):\n", " if weight is None:\n", " return None\n", " expr = f\"(?P\\d*[.,]?\\d*)\\s+(?P[a-zA-Z]*)\"\n", "\n", " # strip is needed to prevent the regex from lazily\n", " # matching just from the first whitespace separator,\n", " # this could happen because the number part in the\n", " # expression is technically all optional, to avoid\n", " # an expression too complex and unreadable\n", " if match := re.search(expr, weight.strip()):\n", " value = float(match.group(\"value\"))\n", " unit = match.group(\"unit\").lower()\n", " return units_to_g(value, unit)\n", "\n", " return weight\n", "\n", "def calculate_dimensional_weight(dimensions: dict[str,float]):\n", " \"\"\"The dimensional weight (in kg) is calculated as\n", " Length * Height * Width (in cm) / 5000.\n", " We'll return it in g here\"\"\"\n", " if dimensions is None:\n", " return None\n", " \n", " height = dimensions.get(\"height\")\n", " width = dimensions.get(\"width\")\n", " depth = dimensions.get(\"depth\")\n", " if None in [height, width, depth]:\n", " return None\n", " \n", " return height * width * depth / 5000\n", "\n", "weight = data[\"raw_specifications\"].apply(parse_value, value=\"weight\").apply(parse_weight)\n", "\n", "dimensional_weight = list(map(calculate_dimensional_weight, dimensions))\n", "\n", "sel = weight.isnull()\n", "weight[sel] = pd.Series(dimensional_weight)[sel]\n", "\n", "weight" ] }, { "cell_type": "code", "execution_count": 74, "id": "49c7495f-1f36-4dc9-8686-de3af7b230a9", "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGdCAYAAACyzRGfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAxjklEQVR4nO3de3BUZZ7/8U9jQidgiFwknSwEwhojEGEwsBAUE0XCgLK6uDuOIsKOTslwN0WlDPxhcEqCFMNGFoVBkcuyiDsbUKZATKaGBBVQCKHkZsQ1QzKYmI0LCdeEy/P7w0n/aHKBTrqfpOP7VXWqPOc855zv04d+/OT0Od0OY4wRAACAJR1auwAAAPDTQvgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYFVQaxdwo2vXrum7775TWFiYHA5Ha5cD/CQZY3T27FlFRUWpQ4fA+BuFsQNoXd6MG20ufHz33Xfq3bt3a5cBQFJpaal69erV2mXcEsYOoG24lXGjzYWPsLAwST8W36VLl1auBvhpqq6uVu/evd3vx0DA2AG0Lm/GjTYXPuoul3bp0oUBBGhlgfTxBWMH0DbcyrgRGB/mAgCAdoPwAQAArCJ8AAAAqwgfAADAKq/Dx6lTp/Tss8+qe/fu6tSpk372s5+poKDAvd4Yo4yMDEVFRSk0NFTJyck6evSoT4sGAACBy6vwcfr0ad1///0KDg7WRx99pGPHjul3v/ud7rjjDnebJUuWaNmyZVqxYoX2798vl8ulMWPG6OzZs76uHQAABCCvHrV9/fXX1bt3b61du9a9rG/fvu7/NsYoKytLCxYs0MSJEyVJ69evV0REhDZt2qQXX3zRN1UDAICA5dWVj23btmno0KH6l3/5F/Xs2VNDhgzR22+/7V5fXFys8vJypaSkuJc5nU4lJSVpz549De6zpqZG1dXVHhMAAGi/vAof3377rVauXKnY2Fh9/PHHmjZtmmbPnq0NGzZIksrLyyVJERERHttFRES4190oMzNT4eHh7omvRwYAoH3zKnxcu3ZN9913nxYtWqQhQ4boxRdf1K9//WutXLnSo92N325mjGn0G8/S09NVVVXlnkpLS73sAgAACCRehY/IyEgNGDDAY1n//v1VUlIiSXK5XJJU7ypHRUVFvashdZxOp/vrkPlaZCDwrVy5UoMGDXK/nxMTE/XRRx81uU1+fr4SEhIUEhKifv36adWqVZaqBdAavAof999/v4qKijyWff311+rTp48kKSYmRi6XS7m5ue71tbW1ys/P18iRI31QLoC2rlevXlq8eLEOHDigAwcO6OGHH9bjjz/e6CP3xcXFGj9+vEaNGqXCwkLNnz9fs2fPVnZ2tuXKAVhjvPDFF1+YoKAg89prr5kTJ06Y//zP/zSdOnUyGzdudLdZvHixCQ8PN1u2bDGHDx82Tz/9tImMjDTV1dW3dIyqqiojyVRVVXlTGgAf8vX7sGvXruadd95pcF1aWpq55557PJa9+OKLZsSIEV4dg7EDaF3evAe9etR22LBh2rp1q9LT0/Xqq68qJiZGWVlZmjRpkrtNWlqaLl68qOnTp+v06dMaPny4cnJy/PbT3CUlJaqsrHTP9+jRQ9HR0X45FgDvXL16VX/4wx90/vx5JSYmNthm7969Hk/ISdLYsWO1Zs0aXb58WcHBwQ1uV1NTo5qaGve8L5+Uu3FckRhbAF/yKnxI0mOPPabHHnus0fUOh0MZGRnKyMhoSV23pKSkRHFx/XXp0gX3spCQTioqOs4gAbSiw4cPKzExUZcuXdLtt9+urVu31rtfrE55eXmDT8hduXJFlZWVioyMbHC7zMxMLVy40Oe1NzSuSIwtgC8F9G+7VFZW/m2A2CipQNJGXbp0od5fLADsiouL06FDh7Rv3z795je/0ZQpU3Ts2LFG2zf0hFxDy6/nryfl6o8rjC2Ar3l95aNt6i/pvtYuAsDfdOzYUXfddZckaejQodq/f7/eeOMN/f73v6/X1uVyNfiEXFBQkLp3797oMZxOp5xOp28L98C4AvhLQF/5ABAYjDEe92dcLzEx0eMJOUnKycnR0KFDG73fA0BgI3wA8Kn58+frk08+0V/+8hcdPnxYCxYsUF5envvG9PT0dD333HPu9tOmTdPJkyeVmpqq48eP691339WaNWs0b9681uoCAD9rJx+7AGgrvv/+e02ePFllZWUKDw/XoEGDtHPnTo0ZM0aSVFZW5v5iQunH7wfasWOHXnrpJb355puKiorS8uXL9eSTT7ZWFwD4GeEDgE+tWbOmyfXr1q2rtywpKUkHDx70U0UA2ho+dgEAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABgFeEDAABYRfgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABgFeEDAABYRfgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABgFeEDAABYRfgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABglVfhIyMjQw6Hw2NyuVzu9cYYZWRkKCoqSqGhoUpOTtbRo0d9XjQAAAhcXl/5GDhwoMrKytzT4cOH3euWLFmiZcuWacWKFdq/f79cLpfGjBmjs2fP+rRoAAAQuLwOH0FBQXK5XO7pzjvvlPTjVY+srCwtWLBAEydOVHx8vNavX68LFy5o06ZNPi8cAAAEJq/Dx4kTJxQVFaWYmBj98pe/1LfffitJKi4uVnl5uVJSUtxtnU6nkpKStGfPnkb3V1NTo+rqao8JAAC0X16Fj+HDh2vDhg36+OOP9fbbb6u8vFwjR47UDz/8oPLycklSRESExzYRERHudQ3JzMxUeHi4e+rdu3czugEAAAKFV+Fj3LhxevLJJ3XvvffqkUce0fbt2yVJ69evd7dxOBwe2xhj6i27Xnp6uqqqqtxTaWmpNyUBaGMyMzM1bNgwhYWFqWfPnnriiSdUVFTU5DZ5eXn1bmZ3OBz66quvLFUNwKYWPWrbuXNn3XvvvTpx4oT7qZcbr3JUVFTUuxpyPafTqS5dunhMAAJXfn6+ZsyYoX379ik3N1dXrlxRSkqKzp8/f9Nti4qKPG5oj42NtVAxANuCWrJxTU2Njh8/rlGjRikmJkYul0u5ubkaMmSIJKm2tlb5+fl6/fXXfVIsgLZv586dHvNr165Vz549VVBQoAcffLDJbXv27Kk77rjDj9UBaAu8uvIxb9485efnq7i4WJ9//rn++Z//WdXV1ZoyZYocDofmzp2rRYsWaevWrTpy5IimTp2qTp066ZlnnvFX/QDauKqqKklSt27dbtp2yJAhioyM1OjRo7Vr164m23KzOhC4vLry8de//lVPP/20Kisrdeedd2rEiBHat2+f+vTpI0lKS0vTxYsXNX36dJ0+fVrDhw9XTk6OwsLC/FI8gLbNGKPU1FQ98MADio+Pb7RdZGSkVq9erYSEBNXU1Og//uM/NHr0aOXl5TV6tSQzM1MLFy70V+kA/Mir8LF58+Ym1zscDmVkZCgjI6MlNQFoJ2bOnKkvv/xSn376aZPt4uLiFBcX555PTExUaWmpli5d2mj4SE9PV2pqqnu+urqap+WAAMFvuwDwi1mzZmnbtm3atWuXevXq5fX2I0aM0IkTJxpdz83qQOBq0Q2nAHAjY4xmzZqlrVu3Ki8vTzExMc3aT2FhoSIjI31cHYC2gPABwKdmzJihTZs26cMPP1RYWJj78fvw8HCFhoZK+vEjk1OnTmnDhg2SpKysLPXt21cDBw5UbW2tNm7cqOzsbGVnZ7daPwD4D+EDgE+tXLlSkpScnOyxfO3atZo6daokqaysTCUlJe51tbW1mjdvnk6dOqXQ0FANHDhQ27dv1/jx422VDcAiwgcAnzLG3LTNunXrPObT0tKUlpbmp4oAtDXccAoAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAqhaFj8zMTDkcDs2dO9e9zBijjIwMRUVFKTQ0VMnJyTp69GhL6wQAAO1Es8PH/v37tXr1ag0aNMhj+ZIlS7Rs2TKtWLFC+/fvl8vl0pgxY3T27NkWFwsAAAJfs8LHuXPnNGnSJL399tvq2rWre7kxRllZWVqwYIEmTpyo+Ph4rV+/XhcuXNCmTZt8VjQAAAhczQofM2bM0KOPPqpHHnnEY3lxcbHKy8uVkpLiXuZ0OpWUlKQ9e/a0rFIAANAueB0+Nm/erIMHDyozM7PeuvLycklSRESEx/KIiAj3uhvV1NSourraYwIQuDIzMzVs2DCFhYWpZ8+eeuKJJ1RUVHTT7fLz85WQkKCQkBD169dPq1atslAtgNbgVfgoLS3VnDlztHHjRoWEhDTazuFweMwbY+otq5OZmanw8HD31Lt3b29KAtDG5Ofna8aMGdq3b59yc3N15coVpaSk6Pz5841uU1xcrPHjx2vUqFEqLCzU/PnzNXv2bGVnZ1usHIAtQd40LigoUEVFhRISEtzLrl69qt27d2vFihXuv27Ky8sVGRnpblNRUVHvakid9PR0paamuuerq6sJIEAA27lzp8f82rVr1bNnTxUUFOjBBx9scJtVq1YpOjpaWVlZkqT+/fvrwIEDWrp0qZ588kl/lwzAMq+ufIwePVqHDx/WoUOH3NPQoUM1adIkHTp0SP369ZPL5VJubq57m9raWuXn52vkyJEN7tPpdKpLly4eE4D2o6qqSpLUrVu3Rtvs3bvX414xSRo7dqwOHDigy5cv+7U+APZ5deUjLCxM8fHxHss6d+6s7t27u5fPnTtXixYtUmxsrGJjY7Vo0SJ16tRJzzzzjO+qBhAQjDFKTU3VAw88UG/suF55eXmD94pduXJFlZWVHldS69TU1KimpsY935L7xUpKSlRZWSlJOn78eKPtrl/Xo0cPRUdHN/uYwE+ZV+HjVqSlpenixYuaPn26Tp8+reHDhysnJ0dhYWG+PhSANm7mzJn68ssv9emnn960bUP3ijW0vE5mZqYWLlzY4hpLSkoUF9dfly5daKJVmaQOevbZZ91LQkI6qajoOAEEaIYWf716Xl6e+3Na6ceBIiMjQ2VlZbp06ZLy8/Ob/IsHQPs0a9Ysbdu2Tbt27VKvXr2abOtyueo9EVdRUaGgoCB17969wW3S09NVVVXlnkpLS5tVZ2Vl5d+Cx0ZJBZJ+20CrM5KuXddmoy5duuC+WgLAOz6/8gHgp80Yo1mzZmnr1q3Ky8tTTEzMTbdJTEzUH//4R49lOTk5Gjp0qIKDgxvcxul0yul0+qTmH/WXdJ+kxj92+f9tALQEPywHwKdmzJihjRs3atOmTQoLC1N5ebnKy8t18eJFd5v09HQ999xz7vlp06bp5MmTSk1N1fHjx/Xuu+9qzZo1mjdvXmt0AYCfET4A+NTKlStVVVWl5ORkRUZGuqf333/f3aasrEwlJSXu+ZiYGO3YsUN5eXn62c9+pt/+9rdavnw5j9kC7RQfuwDwqbobRZuybt26esuSkpJ08OBBP1QEoK3hygcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwyqvwsXLlSg0aNEhdunRRly5dlJiYqI8++si93hijjIwMRUVFKTQ0VMnJyTp69KjPiwYAAIHLq/DRq1cvLV68WAcOHNCBAwf08MMP6/HHH3cHjCVLlmjZsmVasWKF9u/fL5fLpTFjxujs2bN+KR4AAAQer8LHhAkTNH78eN199926++679dprr+n222/Xvn37ZIxRVlaWFixYoIkTJyo+Pl7r16/XhQsXtGnTJn/VDwAAAkyz7/m4evWqNm/erPPnzysxMVHFxcUqLy9XSkqKu43T6VRSUpL27NnT6H5qampUXV3tMQEIbLt379aECRMUFRUlh8OhDz74oMn2eXl5cjgc9aavvvrKTsEArPI6fBw+fFi33367nE6npk2bpq1bt2rAgAEqLy+XJEVERHi0j4iIcK9rSGZmpsLDw91T7969vS0JQBtz/vx5DR48WCtWrPBqu6KiIpWVlbmn2NhYP1UIoDUFebtBXFycDh06pDNnzig7O1tTpkxRfn6+e73D4fBob4ypt+x66enpSk1Ndc9XV1cTQIAAN27cOI0bN87r7Xr27Kk77rjD9wUBaFO8vvLRsWNH3XXXXRo6dKgyMzM1ePBgvfHGG3K5XJJU7ypHRUVFvash13M6ne6nZ+omAD9NQ4YMUWRkpEaPHq1du3Y12ZaPbIHA1eLv+TDGqKamRjExMXK5XMrNzXWvq62tVX5+vkaOHNnSwwBoxyIjI7V69WplZ2dry5YtiouL0+jRo7V79+5Gt+EjWyBwefWxy/z58zVu3Dj17t1bZ8+e1ebNm5WXl6edO3fK4XBo7ty5WrRokWJjYxUbG6tFixapU6dOeuaZZ/xVP4B2IC4uTnFxce75xMRElZaWaunSpXrwwQcb3IaPbIHA5VX4+P777zV58mSVlZUpPDxcgwYN0s6dOzVmzBhJUlpami5evKjp06fr9OnTGj58uHJychQWFuaX4gG0XyNGjNDGjRsbXe90OuV0Oi1WBMBXvAofa9asaXK9w+FQRkaGMjIyWlITAKiwsFCRkZGtXQYAP/D6aRcAuJlz587pm2++cc8XFxfr0KFD6tatm6Kjo5Wenq5Tp05pw4YNkqSsrCz17dtXAwcOVG1trTZu3Kjs7GxlZ2e3VhcA+BHhA4DPHThwQA899JB7vu7ejClTpmjdunUqKytTSUmJe31tba3mzZunU6dOKTQ0VAMHDtT27ds1fvx467UD8D/CBwCfS05OljGm0fXr1q3zmE9LS1NaWpqfqwLQVrT4UVsAAABvED4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABgFeEDAABYRfgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABgFeEDAABYRfgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABgFeEDAABYRfgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVV6Fj8zMTA0bNkxhYWHq2bOnnnjiCRUVFXm0McYoIyNDUVFRCg0NVXJyso4ePerTogEAQODyKnzk5+drxowZ2rdvn3Jzc3XlyhWlpKTo/Pnz7jZLlizRsmXLtGLFCu3fv18ul0tjxozR2bNnfV48AAAIPF6Fj507d2rq1KkaOHCgBg8erLVr16qkpEQFBQWSfrzqkZWVpQULFmjixImKj4/X+vXrdeHCBW3atMkvHQDQ9uzevVsTJkxQVFSUHA6HPvjgg5tuk5+fr4SEBIWEhKhfv35atWqV/wsF0CpadM9HVVWVJKlbt26SpOLiYpWXlyslJcXdxul0KikpSXv27GlwHzU1NaqurvaYAAS28+fPa/DgwVqxYsUttS8uLtb48eM1atQoFRYWav78+Zo9e7ays7P9XCmA1hDU3A2NMUpNTdUDDzyg+Ph4SVJ5ebkkKSIiwqNtRESETp482eB+MjMztXDhwuaWAaANGjdunMaNG3fL7VetWqXo6GhlZWVJkvr3768DBw5o6dKlevLJJ/1UJYDW0uzwMXPmTH355Zf69NNP661zOBwe88aYesvqpKenKzU11T1fXV2t3r17N7csAAFo7969HldMJWns2LFas2aNLl++rODg4Hrb1NTUqKamxj3vzVXTkpISVVZWSpKOHz/ezKo9t62pqZHT6Wx0vkePHoqOjm5Wjc3Z3lf78GZ/vj6eP/jjdW2Nc+3rGnxRkzeaFT5mzZqlbdu2affu3erVq5d7ucvlkvTjFZDIyEj38oqKinpXQ+o4nU6PFwzAT095eXmDV0yvXLmiyspKj/GkTnOvmpaUlCgurr8uXbrQ7HqlMkkd9Oyzz1637DZJVxudDwnppKKi47c0gDdUozfb+2of3uzP18fzB3+9rrbPta9r8EVN3vLqng9jjGbOnKktW7boz3/+s2JiYjzWx8TEyOVyKTc3172strZW+fn5GjlypE8KBtA+NXTFtKHlddLT01VVVeWeSktLb+k4lZWVfxtUN0oqkPTbZlR7RtK1G/ZxtYn5jbp06YLHX5Le1ejd9r7ahzf78/Xx/ME/r6v9c+3rGnxRk7e8uvIxY8YMbdq0SR9++KHCwsLc93iEh4crNDRUDodDc+fO1aJFixQbG6vY2FgtWrRInTp10jPPPOOzogG0Ly6Xyz2e1KmoqFBQUJC6d+/e4DYtv2raX9J9kpr/sUv9fTQ239L9t4Qv9uHN/nx9PH/w5evamufa1zX4oqZb41X4WLlypSQpOTnZY/natWs1depUSVJaWpouXryo6dOn6/Tp0xo+fLhycnIUFhbmk4IBtD+JiYn64x//6LEsJydHQ4cObfB+DwCBzavwUXcZtCkOh0MZGRnKyMhobk0AAty5c+f0zTffuOeLi4t16NAhdevWTdHR0UpPT9epU6e0YcMGSdK0adO0YsUKpaam6te//rX27t2rNWvW6L333mutLgDwo2Y/7QIAjTlw4IAeeugh93zdE21TpkzRunXrVFZWppKSEvf6mJgY7dixQy+99JLefPNNRUVFafny5TxmC7RThA8APpecnNzkldJ169bVW5aUlKSDBw/6sSoAbQW/agsAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrvA4fu3fv1oQJExQVFSWHw6EPPvjAY70xRhkZGYqKilJoaKiSk5N19OhRX9ULAAACnNfh4/z58xo8eLBWrFjR4PolS5Zo2bJlWrFihfbv3y+Xy6UxY8bo7NmzLS4WQOB46623FBMTo5CQECUkJOiTTz5ptG1eXp4cDke96auvvrJYMQBbgrzdYNy4cRo3blyD64wxysrK0oIFCzRx4kRJ0vr16xUREaFNmzbpxRdfbFm1AALC+++/r7lz5+qtt97S/fffr9///vcaN26cjh07pujo6Ea3KyoqUpcuXdzzd955p41yAVjm03s+iouLVV5erpSUFPcyp9OppKQk7dmzx5eHAtCGLVu2TM8//7xeeOEF9e/fX1lZWerdu7dWrlzZ5HY9e/aUy+VyT7fddpuligHY5NPwUV5eLkmKiIjwWB4REeFed6OamhpVV1d7TAACV21trQoKCjz+CJGklJSUm/4RMmTIEEVGRmr06NHatWuXP8sE0Ir88rSLw+HwmDfG1FtWJzMzU+Hh4e6pd+/e/igJgCWVlZW6evWqV3+EREZGavXq1crOztaWLVsUFxen0aNHa/fu3Y0ehz9cgMDl9T0fTXG5XJJ+vAISGRnpXl5RUVFvIKqTnp6u1NRU93x1dTUBBGgHvPkjJC4uTnFxce75xMRElZaWaunSpXrwwQcb3CYzM1MLFy70XcEArPHplY+YmBi5XC7l5ua6l9XW1io/P18jR45scBun06kuXbp4TAACV48ePXTbbbfVu8rR1B8hDRkxYoROnDjR6Pr09HRVVVW5p9LS0mbXDMAur698nDt3Tt988417vri4WIcOHVK3bt0UHR2tuXPnatGiRYqNjVVsbKwWLVqkTp066ZlnnvFp4QDapo4dOyohIUG5ubn6p3/6J/fy3NxcPf7447e8n8LCQo8rqDdyOp1yOp0tqhVA6/A6fBw4cEAPPfSQe77uI5MpU6Zo3bp1SktL08WLFzV9+nSdPn1aw4cPV05OjsLCwnxXNYA2LTU1VZMnT9bQoUOVmJio1atXq6SkRNOmTZP041WLU6dOacOGDZKkrKws9e3bVwMHDlRtba02btyo7OxsZWdnt2Y3APiJ1+EjOTlZxphG1zscDmVkZCgjI6MldQEIYE899ZR++OEHvfrqqyorK1N8fLx27NihPn36SJLKyspUUlLibl9bW6t58+bp1KlTCg0N1cCBA7V9+3aNHz++tboAwI98esMpANSZPn26pk+f3uC6devWecynpaUpLS3NQlUA2gJ+WA4AAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABgFeEDAABYRfgAAABWtcuvVz9+/Lj7v3v06KHo6OhWrAYAAFyvnYWPMkkd9Oyzz7qXhIR0UlHRcQIIAABtRDv72OWMpGuSNkoqkLRRly5dUGVlZatWBQAA/r92duWjTn9J97V2EQAAoAHt7MoHAABo6wgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCqnX7Phye+bh0AgLajnYcPvm4dAIC2pp1/7HJGfN06AABtSzu/8lGHr1sHAKCtaOdXPgAAQFtD+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABg1U/kS8Y88VsvAAC0np9Y+OC3XgAAaG0/sY9dzojfegEAoHX9xK581OG3XgAAaC0/sSsfAACgtRE+AACAVT/Rj13at5KSknr3sfBUT303vk68RgBgB+GjnSkpKVFcXH9dunTBYzlP9Xhq6HXiNQIAO/jYpZ2prKz82/9Q657o4amehtR/nXiNAMAWrny0WzzRc2t4nQDANq58AAAAqwgfAADAKj52uUFznoC42TY3rq+pqZHT6Wx0/mbb++KpDG/3ebM+eFuTL17nlryO1/++T0t4e+5b44matlADAFyP8HGd5jwBcbNtGn765DZJVxudv9n2LX0qw9t93kofvKnJV69zS1/HlmrOubf9RE1bqAEAbsTHLtdpzhMQN9um/vrf6sf/QTY2f7PtW/5Uhrf7vHkfvKvJN69zS1/H395SrS3pR1t4oqYt1AAAN/Jb+HjrrbcUExOjkJAQJSQk6JNPPvHXofyg7gmI/j7cpm59zE3mb7a9NzXdjLf79LZmXx+/qRq8fR1jGlnfHK1x7rxlvwZvx4D8/HwlJCQoJCRE/fr106pVqyxVCsA2v4SP999/X3PnztWCBQtUWFioUaNGady4cSopKfHH4QC0Md6OAcXFxRo/frxGjRqlwsJCzZ8/X7Nnz1Z2drblygHY4JfwsWzZMj3//PN64YUX1L9/f2VlZal3795auXKlPw4HoI3xdgxYtWqVoqOjlZWVpf79++uFF17Qr371Ky1dutRy5QBs8PkNp7W1tSooKNDLL7/ssTwlJUV79uyp176mpkY1NTXu+aqqKklSdXX1TY917ty5v/1XgaRzko57OV/041xBgc6dO6eioqIm10tShw4ddO3aNXcNN9um/nr/1lS/vfywz6Zrbuh1anp/9bf3vgZvX8eW9eFW+tGcf0++nm+shnPnzjX5HqtbZ4xptE1jvB0DJGnv3r1KSUnxWDZ27FitWbNGly9fVnBwcL1tmjt2eD9u6BbaePfvUWr5e+TG7X2xj5bU1Jz3te15X9TYGuPhzd/XLR/PbqUmn44bxsdOnTplJJnPPvvMY/lrr71m7r777nrtX3nlFSOJiYmpDU6lpaV+HwOMMSY2Nta89tprHss+++wzI8l89913DW7D2MHE1DanWxk3/PaorcPh8Jg3xtRbJknp6elKTU11z1+7dk3/93//p+7duzfY/nrV1dXq3bu3SktL1aVLF98Ublmg9yHQ65foQ0OMMTp79qyioqKavY9bHQOaat/Q8jrNGTvaw7luSHvtl0TfAok344bPw0ePHj102223qby83GN5RUWFIiIi6rV3Op0eXwwlSXfccYdXx+zSpUvAn7hA70Og1y/RhxuFh4c3aztvxwBJcrlcDbYPCgpS9+7dG9ymJWNHezjXDWmv/ZLoW6C41XHD5zecduzYUQkJCcrNzfVYnpubq5EjR/r6cADamOaMAYmJifXa5+TkaOjQoQ3e7wEgsPnlaZfU1FS98847evfdd3X8+HG99NJLKikp0bRp0/xxOABtzM3GgPT0dD333HPu9tOmTdPJkyeVmpqq48eP691339WaNWs0b9681uoCAD/yyz0fTz31lH744Qe9+uqrKisrU3x8vHbs2KE+ffr49DhOp1OvvPJKvUuvgSTQ+xDo9Uv0wR9uNgaUlZV5fOdHTEyMduzYoZdeeklvvvmmoqKitHz5cj355JM+rautvU6+0l77JdG39sphTDOepQMAAGgmftsFAABYRfgAAABWET4AAIBVhA8AAGBVwIYPb3+u21d2796tCRMmKCoqSg6HQx988IHHemOMMjIyFBUVpdDQUCUnJ+vo0aMebWpqajRr1iz16NFDnTt31j/+4z/qr3/9q0eb06dPa/LkyQoPD1d4eLgmT56sM2fOeLQpKSnRhAkT1LlzZ/Xo0UOzZ89WbW1tk/VnZmZq2LBhCgsLU8+ePfXEE09c9z3+gdGHlStXatCgQe4v5klMTNRHH30UMPXfKDMzUw6HQ3Pnzg3YPrQWb8eB/Px8JSQkKCQkRP369dOqVavqtcnOztaAAQPkdDo1YMAAbd261V/lN8mbvm3ZskVjxozRnXfe6X5PfPzxxx5t1q1bJ4fDUW+6dOmSv7tSjzd9y8vLa7Dur776yqNdWzhv3vRr6tSpDfZr4MCB7jZt6Zz53E2/gL0N2rx5swkODjZvv/22OXbsmJkzZ47p3LmzOXnypN+PvWPHDrNgwQKTnZ1tJJmtW7d6rF+8eLEJCwsz2dnZ5vDhw+app54ykZGRprq62t1m2rRp5u/+7u9Mbm6uOXjwoHnooYfM4MGDzZUrV9xtfv7zn5v4+HizZ88es2fPHhMfH28ee+wx9/orV66Y+Ph489BDD5mDBw+a3NxcExUVZWbOnNlk/WPHjjVr1641R44cMYcOHTKPPvqoiY6ONufOnQuYPmzbts1s377dFBUVmaKiIjN//nwTHBxsjhw5EhD1X++LL74wffv2NYMGDTJz5swJmHPQFng7Dnz77bemU6dOZs6cOebYsWPm7bffNsHBwea///u/3W327NljbrvtNrNo0SJz/Phxs2jRIhMUFGT27dtnq1vGGO/7NmfOHPP666+bL774wnz99dcmPT3dBAcHm4MHD7rbrF271nTp0sWUlZV5TLZ527ddu3YZSaaoqMij7uv/nbeF8+Ztv86cOePRn9LSUtOtWzfzyiuvuNu0lXPmDwEZPv7hH/7BTJs2zWPZPffcY15++WWrddwYPq5du2ZcLpdZvHixe9mlS5dMeHi4WbVqlTHmx39wwcHBZvPmze42p06dMh06dDA7d+40xhhz7NgxI8njjbN3714jyXz11VfGmB9DUIcOHcypU6fcbd577z3jdDpNVVXVLfehoqLCSDL5+fkB2wdjjOnatat55513Aqr+s2fPmtjYWJObm2uSkpLc4SOQ+tCavB0H0tLSzD333OOx7MUXXzQjRoxwz//iF78wP//5zz3ajB071vzyl7/0UdW3xhdj3IABA8zChQvd82vXrjXh4eG+KrHZvO1bXfg4ffp0o/tsC+etpeds69atxuFwmL/85S/uZW3lnPlDwH3sUvdz3Tf+/HZTP9dtS3FxscrLyz1qczqdSkpKctdWUFCgy5cve7SJiopSfHy8u83evXsVHh6u4cOHu9uMGDFC4eHhHm3i4+M9fsBn7NixqqmpUUFBwS3XXPcz5N26dQvIPly9elWbN2/W+fPnlZiYGFD1z5gxQ48++qgeeeQRj+WB1IfW0pxxYO/evfXajx07VgcOHNDly5ebbGNzbPHFGHft2jWdPXvW/b6uc+7cOfXp00e9evXSY489psLCQp/VfSta0rchQ4YoMjJSo0eP1q5duzzWtfZ588U5W7NmjR555JF6X8bZ2ufMXwIufFRWVurq1av1fqAqIiKi3g9T2VZ3/KZqKy8vV8eOHdW1a9cm2/Ts2bPe/nv27OnR5sbjdO3aVR07drzl18EYo9TUVD3wwAOKj48PqD4cPnxYt99+u5xOp6ZNm6atW7dqwIABAVP/5s2bdfDgQWVmZtZbFyh9aE3NGQca6mtERISuXLmiysrKJtvYfC18Mcb97ne/0/nz5/WLX/zCveyee+7RunXrtG3bNr333nsKCQnR/fffrxMnTvi0/qY0p2+RkZFavXq1srOztWXLFsXFxWn06NHavXu3u01rn7eWnrOysjJ99NFHeuGFFzyWt4Vz5i9++Xp1G7z9uW6bmlPbjW0aat+cNk2ZOXOmvvzyS3366af11rX1PsTFxenQoUM6c+aMsrOzNWXKFOXn5wdE/aWlpZozZ45ycnIUEhLSaLu23Ie2wtvXqKH2Ny5vK2NLc+t47733lJGRoQ8//NAjfI4YMUIjRoxwz99///2677779O///u9avny57wq/Bd70LS4uTnFxce75xMRElZaWaunSpXrwwQebtU9/aW4N69at0x133KEnnnjCY3lbOme+FnBXPprzc922uFwuSWqyNpfLpdraWp0+fbrJNt9//329/f/v//6vR5sbj3P69Gldvnz5ll6HWbNmadu2bdq1a5d69eoVcH3o2LGj7rrrLg0dOlSZmZkaPHiw3njjjYCov6CgQBUVFUpISFBQUJCCgoKUn5+v5cuXKygoyL1tW+5Da2vOONBQXysqKhQUFKTu3bs32cbma9GSMe7999/X888/r//6r/+q93HejTp06KBhw4ZZ/SvaV+P3iBEjPOpu7fPWkn4ZY/Tuu+9q8uTJ6tixY5NtW+Oc+UvAhY/m/Fy3LTExMXK5XB611dbWKj8/311bQkKCgoODPdqUlZXpyJEj7jaJiYmqqqrSF1984W7z+eefq6qqyqPNkSNHVFZW5m6Tk5Mjp9OphISERms0xmjmzJnasmWL/vznPysmJibg+tBYv2pqagKi/tGjR+vw4cM6dOiQexo6dKgmTZqkQ4cOqV+/fm2+D62tOeNAYmJivfY5OTkaOnSogoODm2xjc2xp7hj33nvvaerUqdq0aZMeffTRmx7HGKNDhw4pMjKyxTXfKl+N34WFhR51t/Z5a0m/8vPz9c033+j555+/6XFa45z5jY27Wn2t7pGmNWvWmGPHjpm5c+eazp07e9wl7C9nz541hYWFprCw0Egyy5YtM4WFhe7HqRYvXmzCw8PNli1bzOHDh83TTz/d4COSvXr1Mn/605/MwYMHzcMPP9zgI5KDBg0ye/fuNXv37jX33ntvg49Ijh492hw8eND86U9/Mr169brpI5K/+c1vTHh4uMnLy/N4dOvChQvuNm29D+np6Wb37t2muLjYfPnll2b+/PmmQ4cOJicnJyDqb8j1T7sEah9su9k48PLLL5vJkye729c9avvSSy+ZY8eOmTVr1tR71Pazzz4zt912m1m8eLE5fvy4Wbx4cas+anurfdu0aZMJCgoyb775psf7+syZM+42GRkZZufOneZ//ud/TGFhofnXf/1XExQUZD7//PM23bd/+7d/M1u3bjVff/21OXLkiHn55ZeNJJOdne1u0xbOm7f9qvPss8+a4cOHN7jPtnLO/CEgw4cxxrz55pumT58+pmPHjua+++5zPyrqb3WPfd04TZkyxRjz42OSr7zyinG5XMbpdJoHH3zQHD582GMfFy9eNDNnzjTdunUzoaGh5rHHHjMlJSUebX744QczadIkExYWZsLCwsykSZPqPWp28uRJ8+ijj5rQ0FDTrVs3M3PmTHPp0qUm62+odklm7dq17jZtvQ+/+tWv3Of+zjvvNKNHj3YHj0CovyE3ho9A7ENraGocmDJliklKSvJon5eXZ4YMGWI6duxo+vbta1auXFlvn3/4wx9MXFycCQ4ONvfcc4/H/+Rs8qZvSUlJTY5Lxhgzd+5cEx0d7X7fpKSkmD179ljs0f/nTd9ef/118/d///cmJCTEdO3a1TzwwANm+/bt9fbZFs6bt/8ez5w5Y0JDQ83q1asb3F9bOme+5jDmb3dcAQAAWBBw93wAAIDARvgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABg1f8DfLxSTX485D4AAAAASUVORK5CYII=", "text/plain": [ "
    " ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "ax = plt.subplot(1, 2, 1)\n", "plt.hist(weight, color='blue', edgecolor='black', bins=50)\n", "ax = plt.subplot(1, 2, 2)\n", "plt.hist(weight[weight <= 1], color='blue', edgecolor='black', bins=50)\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": 75, "id": "e6a61e8d-5c55-4312-a48a-a168817e7d77", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0 None\n", "1 [cardboard]\n", "2 None\n", "3 [cardboard]\n", "4 [metal]\n", " ... \n", "162 [plastic]\n", "163 None\n", "164 [fabric]\n", "165 None\n", "166 [stoneware]\n", "Name: raw_specifications, Length: 167, dtype: object" ] }, "execution_count": 75, "metadata": {}, "output_type": "execute_result" } ], "source": [ "materials = data[\"raw_specifications\"].apply(parse_value, value=\"materials\")\n", "\n", "# scoreable materials are:\n", "# * metal\n", "# * wood\n", "# * glass\n", "# * resin\n", "# * fabric\n", "# * plastic\n", "\n", "def material_classifier(material: str) -> str:\n", " \"\"\"I will to match materials to one of the scoreable ones:\n", " * metal\n", " * wood\n", " * glass\n", " * resin\n", " * fabric\n", " * plastic\n", " I found a few, like stoneware and cardboard that I can't fit\n", " there, they'll have to remain unscored for now\"\"\"\n", "\n", " mapping = {\n", " \"polyester\": \"fabric\",\n", " \"spandex\": \"fabric\",\n", " \"leather\": \"fabric\",\n", " \"cardboard\": \"carboard\",\n", " \"crystal\": \"glass\",\n", " \"hardwood\": \"wood\",\n", " \"plywood\": \"wood\",\n", " \"mdf\": \"wood\",\n", " \"wood\": \"wood\",\n", " \"steel\": \"metal\",\n", " \"polycarbonate\": \"plastic\",\n", " \"polypropylene\": \"plastic\",\n", " \"pvc\": \"plastic\",\n", " \"resin\": \"plastic\",\n", " \"stoneware\": \"stoneware\",\n", " \"cardboard\": \"cardboard\",\n", " \"paper\": \"cardboard\",\n", " }\n", " for key, value in mapping.items():\n", " if key in material:\n", " return value\n", " return material\n", "\n", "def clean_material_name(material: str) -> str:\n", " no_paren_annotations = re.sub(\"\\(.*\\)\", \"\", material)\n", " no_amounts = re.sub(\"\\d+%?\", \"\", no_paren_annotations)\n", " return no_amounts.strip().lower()\n", " \n", "def parse_materials(materials: str):\n", " if materials is None:\n", " return\n", " material_ls = [\n", " material_classifier(clean_material_name(x))\n", " for x in materials.split(\",\")\n", " ]\n", " return list(set(material_ls))\n", "\n", "clean_materials = materials.apply(parse_materials)\n", "clean_materials" ] }, { "cell_type": "code", "execution_count": 21, "id": "f6fcaae9-9c24-450c-8737-8b6aa0e406a9", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0 1\n", "1 1\n", "2 1\n", "3 24\n", "4 1\n", " ... \n", "162 1\n", "163 1\n", "164 1\n", "165 1\n", "166 1\n", "Name: raw_specifications, Length: 167, dtype: object" ] }, "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], "source": [ "packaging = data[\"raw_specifications\"].apply(parse_value, value=\"packaging\")\n", "packaging[packaging.isnull()] = 1\n", "packaging" ] }, { "cell_type": "code", "execution_count": 41, "id": "f6ed2736-26d2-403f-9ab0-a228057176e4", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{NoneType, dict}" ] }, "execution_count": 41, "metadata": {}, "output_type": "execute_result" } ], "source": [ "def clean_origin_name(origin: str) -> str:\n", " mapping = {\n", " \"assem usa w/foreign/dom. parts\": \"mixed\",\n", " \"imported\": \"imported\",\n", " \"made in the usa\": \"usa\",\n", " \"made in the usa or imported\": \"mixed\",\n", " }\n", " origin = origin.lower().strip()\n", " return mapping[origin]\n", "\n", "origin = data[\"raw_specifications\"].apply(parse_value, value=\"origin\")\n", "clean_origin = origin.apply(clean_origin_name)\n", "clean_origin" ] }, { "cell_type": "code", "execution_count": 46, "id": "d2df0c7e-cd03-4628-ae29-fae4f0f37dfa", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
    \n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
    materialspackagingoriginweightheightwidthdepthtcinprimary_category
    4[metal]1imported2109.20280058.4202.54058.42086345566Home
    5[glass]1mixed0.08029720.3204.4454.44575552641Holiday Shop
    6[wood]1mixed0.27530313.33510.16010.16081632725Holiday Shop
    11[fabric]1mixed2.72680733.02020.32020.32081726538Holiday Shop
    15[metal]1imported1360.77600074.9301.27031.11586435324Home
    \n", "
    " ], "text/plain": [ " materials packaging origin weight height width depth \\\n", "4 [metal] 1 imported 2109.202800 58.420 2.540 58.420 \n", "5 [glass] 1 mixed 0.080297 20.320 4.445 4.445 \n", "6 [wood] 1 mixed 0.275303 13.335 10.160 10.160 \n", "11 [fabric] 1 mixed 2.726807 33.020 20.320 20.320 \n", "15 [metal] 1 imported 1360.776000 74.930 1.270 31.115 \n", "\n", " tcin primary_category \n", "4 86345566 Home \n", "5 75552641 Holiday Shop \n", "6 81632725 Holiday Shop \n", "11 81726538 Holiday Shop \n", "15 86435324 Home " ] }, "execution_count": 46, "metadata": {}, "output_type": "execute_result" } ], "source": [ "def get_val(xs, i):\n", " if xs is None:\n", " return None\n", " return xs.get(i)\n", "\n", "clean_data = pd.DataFrame(\n", " data={\n", " \"materials\": clean_materials,\n", " \"packaging\": packaging,\n", " \"origin\": clean_origin,\n", " \"weight\": weight,\n", " \"height\": dimensions.apply(get_val, i=\"height\"),\n", " \"width\": dimensions.apply(get_val, i=\"width\"),\n", " \"depth\": dimensions.apply(get_val, i=\"depth\"),\n", " \"tcin\": data[\"raw_specifications\"].apply(parse_value, value=\"tcin\"),\n", " \"primary_category\": data[\"primary_category\"]\n", " }\n", ")\n", "\n", "clean_data\n", "\n", "clean_data[~clean_data.isnull().any(axis=1)].head()" ] }, { "cell_type": "code", "execution_count": 15, "id": "427c5d6a-8f8b-4441-89c5-193d647b49e2", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "4 23271231140\n", "Name: gtin13, dtype: int64" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "weight = data[\"raw_specifications\"].apply(parse_value, value=\"weight\")\n", "sel = weight == \" 4.65 pounds\"\n", "data.loc[sel, \"gtin13\"]" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.4" } }, "nbformat": 4, "nbformat_minor": 5 }