From e8f71da7eab1f68223bb11a5bd92616ddecc6b09 Mon Sep 17 00:00:00 2001 From: Michael Boesherz Date: Wed, 11 May 2022 12:02:06 +0200 Subject: [PATCH] add fields for emission-fuel-consumption and wtlp --- README.md | 9 ++- composer.json | 3 +- src/EventListener/Importer/AdListener.php | 93 ++++++++++++++++++----- src/Resources/contao/dca/tl_mobile_ad.php | 52 +++++++++++++ src/Util/DataUtil.php | 52 +++++++++++++ 5 files changed, 185 insertions(+), 24 deletions(-) create mode 100644 src/Util/DataUtil.php diff --git a/README.md b/README.md index be3bd07..96397c6 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,12 @@ parameters: Usage -- - You can either start the import via `System › Maintenance › Import ads from mobile.de` or the command for the Contao console `contao:mobileads:import`. -At the start of the import all Ads in the Contao database will be set to `published = ''` and all active ads found on mobile.de will then be published again. \ No newline at end of file +At the start of the import all Ads in the Contao database will be set to `published = ''` and all active ads found on mobile.de will then be published again. + +Helper +-- +Functions added to helper converting the data: +- `numero2\MobileDeAdsBundle\Util\DataUtil::wikitext2Html($text);` will convert the wikitext stored in `description_enriched` to html markup +- `numero2\MobileDeAdsBundle\Util\DataUtil::convertKwToPS($kw);` will convert the kw to ps diff --git a/composer.json b/composer.json index 9f8d821..69c3b75 100644 --- a/composer.json +++ b/composer.json @@ -4,7 +4,8 @@ "description": "Import of ads from mobile.de", "license": "LGPL-3.0+", "require": { - "contao/core-bundle": "^4.9" + "contao/core-bundle": "^4.9", + "softark/creole": "^1.2" }, "require-dev": { "contao/manager-plugin": "^2.0" diff --git a/src/EventListener/Importer/AdListener.php b/src/EventListener/Importer/AdListener.php index 530062c..4184d13 100644 --- a/src/EventListener/Importer/AdListener.php +++ b/src/EventListener/Importer/AdListener.php @@ -28,7 +28,12 @@ class AdListener { /** * @var string */ - const API_ENDPOINT = "https://services.mobile.de/search-api/"; + const API_SEARCH_ENDPOINT = "https://services.mobile.de/search-api/"; + + /** + * @var string + */ + const API_SELLER_ENDPOINT = "https://services.mobile.de/seller-api/"; /** * @var Contao\CoreBundle\Framework\ContaoFramework @@ -80,7 +85,7 @@ public function getAvailableAds( $size=100 ): array { for( $page = 1; $page <= $maxPages; $page++ ) { $aParams['page.number'] = $page; - $url = self::API_ENDPOINT . 'search?'.str_replace("&",'&',http_build_query($aParams)); + $url = self::API_SEARCH_ENDPOINT . 'search?'.str_replace("&",'&',http_build_query($aParams)); $oResponse = $oClient->request('GET', $url, $aOptions); $data = []; @@ -102,9 +107,11 @@ public function getAvailableAds( $size=100 ): array { /** * Returns all details for the given ad * - * @param string $id + * @param string $id * * @return array + * + * @see https://services.mobile.de/schema/ad-1.0.xsd */ private function getAdDetails( string $id ): array { @@ -115,7 +122,7 @@ private function getAdDetails( string $id ): array { $oClient = HttpClient::create(); $aOptions = $oOptions->toArray(); - $url = self::API_ENDPOINT . 'ad/'.$id; + $url = self::API_SEARCH_ENDPOINT . 'ad/'.$id; $oResponse = $oClient->request('GET', $url, $aOptions); $data = []; @@ -128,6 +135,45 @@ private function getAdDetails( string $id ): array { } + /** + * Returns all details for the given seller id and ad id + * + * @param string $sellerId + * @param string $adId + * + * @return array + * + * https://services.mobile.de/docs/seller-api.html + */ + private function getAdDetailsForSeller( string $sellerId, string $adId ): array { + + // As we don't have access to this endpoint, we could not yet implement this + return []; + $oOptions = new HttpOptions(); + $oOptions->setAuthBasic($this->username, $this->password); + // $oOptions->setHeaders(['Accept-Language'=>'de', 'Accept' => 'application/vnd.de.mobile.api+json']); + $oOptions->setHeaders(['Accept' => 'application/vnd.de.mobile.api+json']); + + $oClient = HttpClient::create(); + $aOptions = $oOptions->toArray(); + + $url = self::API_SELLER_ENDPOINT . 'sellers/';//.$sellerId.'/ads/'.$adId; + $oResponse = $oClient->request('GET', $url, $aOptions); + // echo $url ."\n"; + $data = []; + + if( $oResponse->getStatusCode() === 200 ) { + $data = $oResponse->getContent(); + // $data = self::parseNamespacedXML($oResponse->getContent()); + // echo print_r($data,1)."\n"; + } + // echo print_r($oResponse,1)."\n"; + // echo $oResponse->getStatusCode()."\n"; + + return $data; + } + + /** * Parses a namespaced XML to an array * @@ -294,6 +340,7 @@ private function importAd( $ad ): int { $aAd = [ 'tstamp' => time() , 'mobile_id' => $ad['@attributes']['key'] + , 'seller_id' => $ad['seller']['@attributes']['key'] , 'creation' => strtotime($ad['creation-date']['@attributes']['value']) , 'modification' => strtotime($ad['modification-date']['@attributes']['value']) , 'url' => $ad['detail-page']['@attributes']['url'] @@ -314,8 +361,24 @@ private function importAd( $ad ): int { , 'door_count' => $this->getValueFromSpecifics($ad['vehicle']['specifics']['door-count']) , 'first_registration' => $this->getValueFromSpecifics($ad['vehicle']['specifics']['first-registration']) , 'emission_class' => $this->getValueFromSpecifics($ad['vehicle']['specifics']['emission-class']) + , 'efc_envkv_compliant' => ((bool)$ad['vehicle']['specifics']['emission-fuel-consumption']['@attributes']['envkv-compliant']) ? '1' : '' + , 'efc_energy_efficiency_class' => $ad['vehicle']['specifics']['emission-fuel-consumption']['@attributes']['energy-efficiency-class'] ?? '' + , 'efc_co2_emission' => $ad['vehicle']['specifics']['emission-fuel-consumption']['@attributes']['co2-emission'] ?? '' + , 'efc_inner' => $ad['vehicle']['specifics']['emission-fuel-consumption']['@attributes']['inner'] ?? '' + , 'efc_outer' => $ad['vehicle']['specifics']['emission-fuel-consumption']['@attributes']['outer'] ?? '' + , 'efc_combined' => $ad['vehicle']['specifics']['emission-fuel-consumption']['@attributes']['combined'] ?? '' + , 'efc_unit' => $ad['vehicle']['specifics']['emission-fuel-consumption']['@attributes']['unit'] ?? '' + , 'efc_petrol_type' => $ad['vehicle']['specifics']['emission-fuel-consumption']['@attributes']['petrol-type'] ?? '' + , 'efc_combined_power_consumption' => $ad['vehicle']['specifics']['emission-fuel-consumption']['@attributes']['combined-power-consumption'] ?? '' , 'emission_sticker' => $this->getValueFromSpecifics($ad['vehicle']['specifics']['emission-sticker']) , 'fuel' => $this->getValueFromSpecifics($ad['vehicle']['specifics']['fuel']) + , 'wltp_consumption_fuel_combined' => $ad['vehicle']['specifics']['wltp-values']['@attributes']['consumption-fuel-combined'] ?? '' + , 'wltp_co2_emission_combined' => $ad['vehicle']['specifics']['wltp-values']['@attributes']['co2-emission-combined'] ?? '' + , 'wltp_consumption_power_combined' => $ad['vehicle']['specifics']['wltp-values']['@attributes']['consumption-power-combined'] ?? '' + , 'wltp_electric_range' => $ad['vehicle']['specifics']['wltp-values']['@attributes']['electric-range'] ?? '' + , 'wltp_consumption_fuel_combined_weighted' => $ad['vehicle']['specifics']['wltp-values']['@attributes']['consumption-fuel-combined-weighted'] ?? '' + , 'wltp_consumption_power_combined_weighted' => $ad['vehicle']['specifics']['wltp-values']['@attributes']['consumption-power-combined-weighted'] ?? '' + , 'wltp_co2_emission_combined_weighted' => $ad['vehicle']['specifics']['wltp-values']['@attributes']['co2-emission-combined-weighted'] ?? '' , 'power' => $this->getValueFromSpecifics($ad['vehicle']['specifics']['power']) , 'gearbox' => $this->getValueFromSpecifics($ad['vehicle']['specifics']['gearbox']) , 'climatisation' => $this->getValueFromSpecifics($ad['vehicle']['specifics']['climatisation']) @@ -340,6 +403,8 @@ private function importAd( $ad ): int { , 'published' => '1' ]; + // $aAdDetailsSeller = $this->getAdDetailsForSeller($aAd['seller_id'], $aAd['mobile_id']); + $oAd = AdModel::findOneBy(['mobile_id=?'], [$aAd['mobile_id']]); if( $oAd && $aAd['modification'] == $oAd->modification ) { @@ -350,8 +415,10 @@ private function importAd( $ad ): int { return $oAd->id; } + // use xml api $aAdDetails = $this->getAdDetails($aAd['mobile_id']); $aAd['description'] = $aAdDetails['description']; + $aAd['description_enriched'] = $aAdDetails['enrichedDescription']; $aAd['images'] = $aAdDetails['images']['image']; foreach( $aAd['images'] as $key => $image) { @@ -367,7 +434,7 @@ private function importAd( $ad ): int { } else { - foreach( $aAd as $field => $value) { + foreach( $aAd as $field => $value ) { $oAd->{$field} = $value; } } @@ -394,20 +461,4 @@ public function truncateTable(): void { public function unpublishAll(): void { $this->connection->query("UPDATE ".AdModel::getTable()." SET published=''"); } - - - /** - * Converts kW to PS - * - * @param float kw - * - * @return float - **/ - public static function convertKwToPS( float $kw ): float { - - if( empty($kw) ) - return 0; - - return ceil($kw*1.358620689655172); - } } diff --git a/src/Resources/contao/dca/tl_mobile_ad.php b/src/Resources/contao/dca/tl_mobile_ad.php index 010dcee..ef12e07 100644 --- a/src/Resources/contao/dca/tl_mobile_ad.php +++ b/src/Resources/contao/dca/tl_mobile_ad.php @@ -89,12 +89,61 @@ , 'emission_class' => [ 'sql' => "varchar(64) NOT NULL default ''" ] + // emission-fuel-consumption + , 'efc_envkv_compliant' => [ + 'sql' => "char(1) NOT NULL default ''" + ] + , 'efc_energy_efficiency_class' => [ + 'sql' => "varchar(64) NOT NULL default ''" + ] + , 'efc_co2_emission' => [ + 'sql' => "varchar(32) NOT NULL default ''" + ] + , 'efc_inner' => [ + 'sql' => "varchar(32) NOT NULL default ''" + ] + , 'efc_outer' => [ + 'sql' => "varchar(32) NOT NULL default ''" + ] + , 'efc_combined' => [ + 'sql' => "varchar(32) NOT NULL default ''" + ] + , 'efc_unit' => [ + 'sql' => "varchar(32) NOT NULL default ''" + ] + , 'efc_petrol_type' => [ + 'sql' => "varchar(32) NOT NULL default ''" + ] + , 'efc_combined_power_consumption' => [ + 'sql' => "varchar(32) NOT NULL default ''" + ] , 'emission_sticker' => [ 'sql' => "varchar(64) NOT NULL default ''" ] , 'fuel' => [ 'sql' => "varchar(128) NOT NULL default ''" ] + , 'wltp_consumption_fuel_combined' => [ + 'sql' => "varchar(32) NOT NULL default ''" + ] + , 'wltp_co2_emission_combined' => [ + 'sql' => "varchar(32) NOT NULL default ''" + ] + , 'wltp_consumption_power_combined' => [ + 'sql' => "varchar(32) NOT NULL default ''" + ] + , 'wltp_electric_range' => [ + 'sql' => "varchar(32) NOT NULL default ''" + ] + , 'wltp_consumption_fuel_combined_weighted' => [ + 'sql' => "varchar(32) NOT NULL default ''" + ] + , 'wltp_consumption_power_combined_weighted' => [ + 'sql' => "varchar(32) NOT NULL default ''" + ] + , 'wltp_co2_emission_combined_weighted' => [ + 'sql' => "varchar(32) NOT NULL default ''" + ] , 'power' => [ 'sql' => "varchar(128) NOT NULL default ''" ] @@ -143,6 +192,9 @@ , 'description' => [ 'sql' => "blob NULL" ] + , 'description_enriched' => [ + 'sql' => "blob NULL" + ] , 'image' => [ 'sql' => "varchar(255) NOT NULL default ''" ] diff --git a/src/Util/DataUtil.php b/src/Util/DataUtil.php new file mode 100644 index 0000000..887b7b3 --- /dev/null +++ b/src/Util/DataUtil.php @@ -0,0 +1,52 @@ + + * @author Michael Bösherz + * @license LGPL + * @copyright Copyright (c) 2021, numero2 - Agentur für digitales Marketing GbR + */ + + +namespace numero2\MobileDeAdsBundle\Util; + +use softark\creole\Creole; + + +class DataUtil { + + + /** + * Converts kW to PS + * + * @param float kw + * + * @return float + **/ + public static function convertKwToPS( float $kw ): float { + + if( empty($kw) ) + return 0; + + return ceil($kw*1.358620689655172); + } + + + /** + * Converts the given text in wikisyntax to html + * + * @param string $text + * + * @return string + **/ + public static function wikitext2Html( string $text ): string { + + $parser = new Creole(); + + $text = html_entity_decode(str_replace('\\\\', "\\\\\r\n", $text)); + + return $parser->parse($text); + } +}