WordPress.org
Get WordPress

Plugin Directory

Changeset 3488445


Ignore:
Timestamp:
03/22/2026 09:45:05 PM (10 days ago)
Author:
bsolveit
Message:

v1.3.0: Add AI Crawler Analytics — bot tracking, discovery file access, robots.txt conflicts, dashboard widgets, log viewer with CSV export

Location:
ai-discovery-files
Files:
24 added
14 edited
1 copied

Legend:

Unmodified
Added
Removed

  • ai-discovery-files/tags/1.3.0/admin/class-admin.php

    r3482407 r3488445  
    3939        add_action( 'wp_ajax_aidf_dismiss_review', array( __CLASS__, 'ajax_dismiss_review' ) );
    4040        add_action( 'wp_ajax_aidf_search_pages', array( __CLASS__, 'ajax_search_pages' ) );
     41        add_action( 'wp_ajax_aidf_crawler_overview', array( __CLASS__, 'ajax_crawler_overview' ) );
     42        add_action( 'wp_ajax_aidf_crawler_bot_detail', array( __CLASS__, 'ajax_crawler_bot_detail' ) );
     43        add_action( 'wp_ajax_aidf_crawler_log_entries', array( __CLASS__, 'ajax_crawler_log_entries' ) );
     44        add_action( 'wp_ajax_aidf_crawler_export_csv', array( __CLASS__, 'ajax_crawler_export_csv' ) );
     45        add_action( 'wp_ajax_aidf_crawler_clear_log', array( __CLASS__, 'ajax_crawler_clear_log' ) );
     46        add_action( 'wp_ajax_aidf_crawler_conflicts', array( __CLASS__, 'ajax_crawler_conflicts' ) );
     47        add_action( 'wp_ajax_aidf_crawler_file_access', array( __CLASS__, 'ajax_crawler_file_access' ) );
     48        add_action( 'wp_ajax_aidf_crawler_save_settings', array( __CLASS__, 'ajax_crawler_save_settings' ) );
     49        add_action( 'wp_dashboard_setup', array( __CLASS__, 'register_dashboard_widgets' ) );
     50        add_action( 'admin_head-index.php', array( __CLASS__, 'dashboard_widget_styles' ) );
    4151        add_action( 'admin_notices', array( __CLASS__, 'maybe_show_welcome_notice' ) );
    4252        add_action( 'admin_notices', array( __CLASS__, 'maybe_show_conflict_notice' ) );
    … …  
    123133                    'noResults'       => __( 'No pages found', 'ai-discovery-files' ),
    124134                    'typeToSearch'    => __( 'Type to search…', 'ai-discovery-files' ),
     135                    'saving'          => __( 'Saving...', 'ai-discovery-files' ),
     136                    'saveSettings'    => __( 'Save Settings', 'ai-discovery-files' ),
     137                    'confirmClearLog' => __( 'This permanently deletes all crawler log data. This cannot be undone. Continue?', 'ai-discovery-files' ),
     138                    'noDataYet'       => __( 'No data for this period.', 'ai-discovery-files' ),
     139                    'justNow'         => __( 'Just now', 'ai-discovery-files' ),
     140                    /* translators: %d: number of minutes ago */
     141                    'mAgo'            => __( '%dm ago', 'ai-discovery-files' ),
     142                    /* translators: %d: number of hours ago */
     143                    'hAgo'            => __( '%dh ago', 'ai-discovery-files' ),
     144                    /* translators: %d: number of days ago */
     145                    'dAgo'            => __( '%dd ago', 'ai-discovery-files' ),
     146                    'loading'         => __( 'Loading...', 'ai-discovery-files' ),
    125147                ),
    126148            )
    127149        );
     150
     151        $current_tab = self::get_current_tab();
     152        if ( 'crawlers' === $current_tab ) {
     153            wp_enqueue_style( 'aidf-crawlers', AIDF_PLUGIN_URL . 'admin/css/crawlers.css', array( 'aidf-admin' ), AIDF_VERSION );
     154            wp_enqueue_script( 'aidf-crawlers', AIDF_PLUGIN_URL . 'admin/js/crawlers.js', array( 'jquery' ), AIDF_VERSION, true );
     155        }
    128156    }
    129157
    … …  
    287315        $tab = isset( $_GET['tab'] ) ? sanitize_text_field( wp_unslash( $_GET['tab'] ) ) : 'identity';
    288316
    289         $valid_tabs = array( 'identity', 'permissions', 'content', 'advanced', 'preview', 'status' );
     317        $valid_tabs = array( 'identity', 'permissions', 'content', 'advanced', 'preview', 'status', 'crawlers' );
    290318
    291319        if ( ! in_array( $tab, $valid_tabs, true ) ) {
    … …  
    309337            'preview'     => __( 'Preview', 'ai-discovery-files' ),
    310338            'status'      => __( 'Status', 'ai-discovery-files' ),
     339            'crawlers'    => __( 'AI Crawlers', 'ai-discovery-files' ),
    311340        );
    312341    }
    … …  
    658687        wp_send_json_success( $results );
    659688    }
     689
     690    /**
     691     * Register WordPress dashboard widgets for crawler analytics.
     692     *
     693     * @since 1.3.0
     694     */
     695    public static function register_dashboard_widgets() {
     696        $settings = AIDF_Plugin::get_settings();
     697        if ( empty( $settings['crawler_logging_enabled'] ) ) {
     698            return;
     699        }
     700
     701        wp_add_dashboard_widget(
     702            'aidf_crawler_activity',
     703            __( 'AI Crawler Activity', 'ai-discovery-files' ),
     704            array( __CLASS__, 'render_crawler_activity_widget' )
     705        );
     706
     707        wp_add_dashboard_widget(
     708            'aidf_file_access',
     709            __( 'AI Discovery File Access', 'ai-discovery-files' ),
     710            array( __CLASS__, 'render_file_access_widget' )
     711        );
     712    }
     713
     714    /**
     715     * Render the Crawler Activity dashboard widget.
     716     *
     717     * @since 1.3.0
     718     */
     719    public static function render_crawler_activity_widget() {
     720        include AIDF_PLUGIN_DIR . 'admin/views/partials/crawler-dashboard-widget.php';
     721    }
     722
     723    /**
     724     * Render the File Access dashboard widget.
     725     *
     726     * @since 1.3.0
     727     */
     728    public static function render_file_access_widget() {
     729        include AIDF_PLUGIN_DIR . 'admin/views/partials/crawler-file-access-widget.php';
     730    }
     731
     732    /**
     733     * Output inline CSS for dashboard widgets (only on the Dashboard screen).
     734     *
     735     * Gated on crawler_logging_enabled so styles are not injected when the
     736     * feature is disabled and the widgets are not registered.
     737     *
     738     * @since 1.3.0
     739     */
     740    public static function dashboard_widget_styles() {
     741        $settings = AIDF_Plugin::get_settings();
     742        if ( empty( $settings['crawler_logging_enabled'] ) ) {
     743            return;
     744        }
     745        ?>
     746        <style>
     747        /* AI Discovery Files — Dashboard Widget Styles */
     748        .aidf-widget-top-bots { color: #5a5d6b; font-size: 13px; }
     749        .aidf-widget-warning { color: #d97706; font-size: 13px; }
     750        .aidf-widget-warning .dashicons { font-size: 16px; width: 16px; height: 16px; vertical-align: text-bottom; }
     751        .aidf-widget-empty { color: #8b8fa3; font-style: italic; }
     752        .aidf-widget-link { text-align: right; margin-bottom: 0; }
     753        .aidf-widget-link a { color: #e77d15; text-decoration: none; font-weight: 500; }
     754        .aidf-widget-link a:hover { color: #d06e0d; }
     755        .aidf-widget-files { display: flex; flex-direction: column; gap: 6px; margin-bottom: 12px; }
     756        .aidf-widget-file-row { display: flex; align-items: center; gap: 8px; font-size: 12px; }
     757        .aidf-widget-file-name { font-family: ui-monospace, 'SF Mono', Menlo, Consolas, monospace; width: 110px; flex-shrink: 0; color: #1a1a2e; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
     758        .aidf-widget-file-bar { flex: 1; height: 8px; background: #e5e7eb; border-radius: 4px; overflow: hidden; }
     759        .aidf-widget-file-bar-fill { display: block; height: 100%; background: #e77d15; border-radius: 4px; transition: width 0.3s ease; }
     760        .aidf-widget-file-count { width: 30px; text-align: right; font-weight: 600; color: #1a1a2e; flex-shrink: 0; }
     761        .aidf-widget-file-bots { width: 90px; flex-shrink: 0; color: #5a5d6b; font-size: 11px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
     762        .aidf-widget-more { color: #8b8fa3; }
     763        .aidf-widget-empty-text { color: #8b8fa3; }
     764        .aidf-widget-summary { color: #5a5d6b; font-size: 13px; }
     765        .aidf-widget-summary strong { color: #1a1a2e; }
     766        </style>
     767        <?php
     768    }
     769
     770    /**
     771     * AJAX: Crawler overview data (summary + chart + breakdown).
     772     *
     773     * @since 1.3.0
     774     */
     775    public static function ajax_crawler_overview() {
     776        check_ajax_referer( 'aidf_admin_nonce', 'nonce' );
     777        if ( ! current_user_can( 'manage_options' ) ) {
     778            wp_send_json_error( __( 'Permission denied.', 'ai-discovery-files' ) );
     779        }
     780
     781        $days = isset( $_GET['days'] ) ? absint( $_GET['days'] ) : 30;
     782        if ( ! in_array( $days, array( 7, 30, 90 ), true ) ) {
     783            $days = 30;
     784        }
     785
     786        wp_send_json_success( array(
     787            'summary'   => AIDF_Crawler_Analytics::get_summary( $days ),
     788            'chart'     => AIDF_Crawler_Analytics::get_chart_data( $days ),
     789            'breakdown' => AIDF_Crawler_Analytics::get_bot_breakdown( $days ),
     790            'days'      => $days,
     791        ) );
     792    }
     793
     794    /**
     795     * AJAX: Crawler bot detail.
     796     *
     797     * Returns detailed analytics for a single bot: daily trend, status code
     798     * breakdown, pages visited, discovery file access, and first/last seen.
     799     *
     800     * @since 1.3.0
     801     */
     802    public static function ajax_crawler_bot_detail() {
     803        check_ajax_referer( 'aidf_admin_nonce', 'nonce' );
     804        if ( ! current_user_can( 'manage_options' ) ) {
     805            wp_send_json_error( __( 'Permission denied.', 'ai-discovery-files' ) );
     806        }
     807
     808        $bot_name = isset( $_GET['bot_name'] ) ? sanitize_text_field( wp_unslash( $_GET['bot_name'] ) ) : '';
     809        if ( empty( $bot_name ) ) {
     810            wp_send_json_error( __( 'Bot name required.', 'ai-discovery-files' ) );
     811        }
     812
     813        $days = isset( $_GET['days'] ) ? absint( $_GET['days'] ) : 30;
     814        if ( ! in_array( $days, array( 7, 30, 90 ), true ) ) {
     815            $days = 30;
     816        }
     817
     818        wp_send_json_success( AIDF_Crawler_Analytics::get_bot_detail( $bot_name, $days ) );
     819    }
     820
     821    /**
     822     * AJAX: Crawler log entries.
     823     *
     824     * Returns a paginated, filtered slice of the raw crawler log.
     825     * Supported filters: bot_name, date_from, date_to, status_code, url_path.
     826     *
     827     * @since 1.3.0
     828     */
     829    public static function ajax_crawler_log_entries() {
     830        check_ajax_referer( 'aidf_admin_nonce', 'nonce' );
     831        if ( ! current_user_can( 'manage_options' ) ) {
     832            wp_send_json_error( __( 'Permission denied.', 'ai-discovery-files' ) );
     833        }
     834
     835        $args = array(
     836            'bot_name'    => isset( $_GET['bot_name'] ) ? sanitize_text_field( wp_unslash( $_GET['bot_name'] ) ) : '',
     837            'date_from'   => isset( $_GET['date_from'] ) ? sanitize_text_field( wp_unslash( $_GET['date_from'] ) ) : '',
     838            'date_to'     => isset( $_GET['date_to'] ) ? sanitize_text_field( wp_unslash( $_GET['date_to'] ) ) : '',
     839            'status_code' => isset( $_GET['status_code'] ) ? sanitize_text_field( wp_unslash( $_GET['status_code'] ) ) : '',
     840            'url_path'    => isset( $_GET['url_path'] ) ? sanitize_text_field( wp_unslash( $_GET['url_path'] ) ) : '',
     841            'page'        => isset( $_GET['paged'] ) ? absint( $_GET['paged'] ) : 1,
     842            'per_page'    => 50,
     843            'orderby'     => isset( $_GET['orderby'] ) ? sanitize_text_field( wp_unslash( $_GET['orderby'] ) ) : 'created_at',
     844            'order'       => isset( $_GET['order'] ) ? sanitize_text_field( wp_unslash( $_GET['order'] ) ) : 'DESC',
     845        );
     846
     847        wp_send_json_success( AIDF_Crawler_Analytics::get_log_entries( $args ) );
     848    }
     849
     850    /**
     851     * AJAX: Export crawler log as CSV.
     852     *
     853     * Streams a UTF-8 CSV file with all matching log entries and exits.
     854     * Filter parameters mirror those of ajax_crawler_log_entries.
     855     *
     856     * @since 1.3.0
     857     */
     858    public static function ajax_crawler_export_csv() {
     859        check_ajax_referer( 'aidf_admin_nonce', 'nonce' );
     860        if ( ! current_user_can( 'manage_options' ) ) {
     861            wp_die( esc_html__( 'Permission denied.', 'ai-discovery-files' ) );
     862        }
     863
     864        $args = array(
     865            'bot_name'    => isset( $_GET['bot_name'] ) ? sanitize_text_field( wp_unslash( $_GET['bot_name'] ) ) : '',
     866            'date_from'   => isset( $_GET['date_from'] ) ? sanitize_text_field( wp_unslash( $_GET['date_from'] ) ) : '',
     867            'date_to'     => isset( $_GET['date_to'] ) ? sanitize_text_field( wp_unslash( $_GET['date_to'] ) ) : '',
     868            'status_code' => isset( $_GET['status_code'] ) ? sanitize_text_field( wp_unslash( $_GET['status_code'] ) ) : '',
     869            'url_path'    => isset( $_GET['url_path'] ) ? sanitize_text_field( wp_unslash( $_GET['url_path'] ) ) : '',
     870        );
     871
     872        AIDF_Crawler_Analytics::export_csv( $args );
     873    }
     874
     875    /**
     876     * AJAX: Clear crawler log.
     877     *
     878     * @since 1.3.0
     879     */
     880    public static function ajax_crawler_clear_log() {
     881        check_ajax_referer( 'aidf_admin_nonce', 'nonce' );
     882        if ( ! current_user_can( 'manage_options' ) ) {
     883            wp_send_json_error( __( 'Permission denied.', 'ai-discovery-files' ) );
     884        }
     885
     886        global $wpdb;
     887
     888        // Use DELETE instead of TRUNCATE for shared hosting compatibility
     889        // (TRUNCATE requires DROP privilege which many hosts don't grant).
     890        // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching
     891        $wpdb->query( "DELETE FROM {$wpdb->prefix}aidf_crawler_log" );
     892        // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching
     893        $wpdb->query( "DELETE FROM {$wpdb->prefix}aidf_crawler_summary" );
     894
     895        wp_send_json_success( array(
     896            'message' => __( 'Log data cleared.', 'ai-discovery-files' ),
     897        ) );
     898    }
     899
     900    /**
     901     * AJAX: Crawler conflicts data.
     902     *
     903     * @since 1.3.0
     904     */
     905    public static function ajax_crawler_conflicts() {
     906        check_ajax_referer( 'aidf_admin_nonce', 'nonce' );
     907        if ( ! current_user_can( 'manage_options' ) ) {
     908            wp_send_json_error( __( 'Permission denied.', 'ai-discovery-files' ) );
     909        }
     910
     911        $conflicts = AIDF_Crawler_Conflicts::get_conflicts();
     912
     913        // Sanitize messages for safe DOM insertion in JavaScript.
     914        foreach ( $conflicts as &$alert ) {
     915            $alert['message'] = wp_kses_post( $alert['message'] );
     916        }
     917        unset( $alert );
     918
     919        wp_send_json_success( array(
     920            'conflicts' => $conflicts,
     921        ) );
     922    }
     923
     924    /**
     925     * AJAX: Crawler file access data.
     926     *
     927     * @since 1.3.0
     928     */
     929    public static function ajax_crawler_file_access() {
     930        check_ajax_referer( 'aidf_admin_nonce', 'nonce' );
     931        if ( ! current_user_can( 'manage_options' ) ) {
     932            wp_send_json_error( __( 'Permission denied.', 'ai-discovery-files' ) );
     933        }
     934
     935        $days = isset( $_GET['days'] ) ? absint( $_GET['days'] ) : 30;
     936        if ( ! in_array( $days, array( 7, 30, 90 ), true ) ) {
     937            $days = 30;
     938        }
     939
     940        wp_send_json_success( array(
     941            'file_access' => AIDF_Crawler_Analytics::get_file_access( $days ),
     942            'days'        => $days,
     943        ) );
     944    }
     945
     946    /**
     947     * AJAX: Save crawler settings.
     948     *
     949     * @since 1.3.0
     950     */
     951    public static function ajax_crawler_save_settings() {
     952        check_ajax_referer( 'aidf_admin_nonce', 'nonce' );
     953        if ( ! current_user_can( 'manage_options' ) ) {
     954            wp_send_json_error( __( 'Permission denied.', 'ai-discovery-files' ) );
     955        }
     956
     957        $settings    = AIDF_Plugin::get_settings();
     958        $was_enabled = ! empty( $settings['crawler_logging_enabled'] );
     959
     960        // Logging toggle.
     961        $settings['crawler_logging_enabled'] = ! empty( $_POST['crawler_logging_enabled'] );
     962
     963        // Retention period.
     964        $valid_retention = array( '30', '60', '90', '180', '365' );
     965        $retention       = isset( $_POST['crawler_log_retention'] )
     966            ? sanitize_text_field( wp_unslash( $_POST['crawler_log_retention'] ) )
     967            : '90';
     968        $settings['crawler_log_retention'] = in_array( $retention, $valid_retention, true ) ? $retention : '90';
     969
     970        // Enabled bots.
     971        $enabled_bots = array();
     972        $raw_bots = isset( $_POST['crawler_enabled_bots'] ) ? array_map( 'sanitize_text_field', wp_unslash( (array) $_POST['crawler_enabled_bots'] ) ) : array();
     973        if ( ! empty( $raw_bots ) ) {
     974            $all_bot_keys = array_keys( AIDF_Crawler_Registry::get_bots() );
     975            foreach ( $raw_bots as $key ) {
     976                if ( in_array( $key, $all_bot_keys, true ) ) {
     977                    $enabled_bots[] = $key;
     978                }
     979            }
     980        }
     981        $settings['crawler_enabled_bots'] = $enabled_bots;
     982
     983        update_option( 'aidf_settings', $settings );
     984
     985        // Manage cron based on toggle change.
     986        $is_enabled = ! empty( $settings['crawler_logging_enabled'] );
     987        if ( $is_enabled && ! $was_enabled ) {
     988            AIDF_Crawler_Logger::schedule_cron();
     989        } elseif ( ! $is_enabled && $was_enabled ) {
     990            AIDF_Crawler_Logger::clear_cron();
     991        }
     992
     993        wp_send_json_success( array(
     994            'message' => __( 'Settings saved.', 'ai-discovery-files' ),
     995        ) );
     996    }
    660997}

  • ai-discovery-files/tags/1.3.0/admin/class-settings.php

    r3475304 r3488445  
    278278        }
    279279
     280        // Preserve crawler analytics settings.
     281        // Read from $input first (AJAX save passes crawler keys via update_option),
     282        // fall back to $fallback (non-crawler tab saves don't include these keys).
     283        $clean['crawler_logging_enabled'] = isset( $input['crawler_logging_enabled'] )
     284            ? (bool) $input['crawler_logging_enabled']
     285            : ( isset( $fallback['crawler_logging_enabled'] ) ? (bool) $fallback['crawler_logging_enabled'] : false );
     286        $clean['crawler_log_retention'] = isset( $input['crawler_log_retention'] )
     287            ? sanitize_text_field( $input['crawler_log_retention'] )
     288            : ( isset( $fallback['crawler_log_retention'] ) ? $fallback['crawler_log_retention'] : '90' );
     289        $clean['crawler_enabled_bots'] = isset( $input['crawler_enabled_bots'] ) && is_array( $input['crawler_enabled_bots'] )
     290            ? array_map( 'sanitize_text_field', $input['crawler_enabled_bots'] )
     291            : ( isset( $fallback['crawler_enabled_bots'] ) ? $fallback['crawler_enabled_bots'] : array() );
     292
    280293        // Flush rewrite rules since active files may have changed.
    281294        AIDF_Server::register_rewrite_rules();

  • ai-discovery-files/tags/1.3.0/admin/views/settings-page.php

    r3475304 r3488445  
    5151            'preview'     => 'dashicons-media-code',
    5252            'status'      => 'dashicons-yes-alt',
     53            'crawlers'    => 'dashicons-chart-bar',
    5354        );
    5455
    … …  
    7778        <?php include AIDF_PLUGIN_DIR . 'admin/views/partials/directory-cta.php'; ?>
    7879
    79         <?php if ( in_array( $current_tab, array( 'preview', 'status' ), true ) ) : ?>
     80        <?php if ( in_array( $current_tab, array( 'preview', 'status', 'crawlers' ), true ) ) : ?>
    8081            <?php include AIDF_PLUGIN_DIR . 'admin/views/tab-' . $current_tab . '.php'; ?>
    8182        <?php else : ?>
    … …  
    8586
    8687                <!-- Persist active_files and spec_attribution across all tabs -->
    87                 <?php if ( 'status' !== $current_tab ) : ?>
     88                <?php if ( ! in_array( $current_tab, array( 'status', 'crawlers' ), true ) ) : ?>
    8889                    <?php
    8990                    $active_files = isset( $settings['active_files'] ) ? (array) $settings['active_files'] : array();

  • ai-discovery-files/tags/1.3.0/ai-discovery-files.php

    r3487801 r3488445  
    33 * Plugin Name:       AI Discovery Files – llms.txt & AI Visibility
    44 * Plugin URI:        https://www.ai-visibility.org.uk/wordpress-plugin/ai-discovery-files/
    5  * Description:       The only WordPress plugin built on the official AI Discovery Files Specification. Generates all 10 files. Built by 365i.
    6  * Version:           1.2.1
     5 * Description:       The only WordPress plugin that generates all 10 AI Discovery Files and tracks which AI bots visit your site. Built by 365i.
     6 * Version:           1.3.0
    77 * Requires at least: 6.2
    88 * Requires PHP:      8.0
    … …  
    2424 * Plugin constants.
    2525 */
    26 define( 'AIDF_VERSION', '1.2.1' );
     26define( 'AIDF_VERSION', '1.3.0' );
    2727define( 'AIDF_PLUGIN_FILE', __FILE__ );
    2828define( 'AIDF_PLUGIN_DIR', plugin_dir_path( __FILE__ ) );
    … …  
    3838require_once AIDF_PLUGIN_DIR . 'includes/class-server.php';
    3939require_once AIDF_PLUGIN_DIR . 'includes/class-validator.php';
     40require_once AIDF_PLUGIN_DIR . 'includes/class-crawler-registry.php';
     41require_once AIDF_PLUGIN_DIR . 'includes/class-crawler-logger.php';
     42require_once AIDF_PLUGIN_DIR . 'includes/class-crawler-analytics.php';
     43require_once AIDF_PLUGIN_DIR . 'includes/class-crawler-conflicts.php';
     44
     45/**
     46 * Create or upgrade the crawler analytics database tables.
     47 *
     48 * Uses dbDelta() so it is safe to call on both fresh installs and upgrades.
     49 * Stores the installed DB version in the aidf_db_version option.
     50 */
     51function aidf_create_crawler_tables() {
     52    global $wpdb;
     53
     54    $charset_collate = $wpdb->get_charset_collate();
     55
     56    $sql = "CREATE TABLE {$wpdb->prefix}aidf_crawler_log (
     57  id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
     58  bot_name varchar(50) NOT NULL,
     59  bot_ua varchar(500) NOT NULL DEFAULT '',
     60  url_path varchar(500) NOT NULL DEFAULT '',
     61  status_code smallint(5) unsigned NOT NULL DEFAULT 200,
     62  created_at datetime NOT NULL,
     63  PRIMARY KEY  (id),
     64  KEY bot_date (bot_name, created_at),
     65  KEY date_idx (created_at)
     66) $charset_collate;
     67
     68CREATE TABLE {$wpdb->prefix}aidf_crawler_summary (
     69  id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
     70  bot_name varchar(50) NOT NULL,
     71  log_date date NOT NULL,
     72  hit_count int(10) unsigned NOT NULL DEFAULT 0,
     73  top_page varchar(500) NOT NULL DEFAULT '',
     74  status_200 int(10) unsigned NOT NULL DEFAULT 0,
     75  status_403 int(10) unsigned NOT NULL DEFAULT 0,
     76  status_other int(10) unsigned NOT NULL DEFAULT 0,
     77  PRIMARY KEY  (id),
     78  UNIQUE KEY bot_day (bot_name, log_date),
     79  KEY date_idx (log_date)
     80) $charset_collate;";
     81
     82    require_once ABSPATH . 'wp-admin/includes/upgrade.php';
     83    dbDelta( $sql );
     84
     85    update_option( 'aidf_db_version', AIDF_VERSION );
     86}
    4087
    4188/**
    … …  
    55102
    56103    set_transient( 'aidf_activation_redirect', true, 30 );
     104
     105    aidf_create_crawler_tables();
    57106}
    58107register_activation_hook( __FILE__, 'aidf_activate' );
    … …  
    63112function aidf_deactivate() {
    64113    flush_rewrite_rules();
     114    AIDF_Crawler_Logger::clear_cron();
    65115}
    66116register_deactivation_hook( __FILE__, 'aidf_deactivate' );
    … …  
    73123}
    74124add_action( 'plugins_loaded', 'aidf_init' );
     125
     126/**
     127 * Cron: prune old crawler log entries.
     128 */
     129add_action( 'aidf_prune_crawler_log', array( 'AIDF_Crawler_Logger', 'prune_log' ) );
     130
     131/**
     132 * Cron: rebuild crawler summary table from raw log data.
     133 */
     134add_action( 'aidf_rebuild_crawler_summary', array( 'AIDF_Crawler_Analytics', 'rebuild_summary' ) );
     135
     136/**
     137 * Run DB migrations when the plugin version has advanced.
     138 *
     139 * Hooked to admin_init so it runs on every admin page load, but only
     140 * performs work when the stored version is behind the current version.
     141 */
     142function aidf_maybe_upgrade_db() {
     143    $installed_version = get_option( 'aidf_db_version', '0' );
     144    if ( version_compare( $installed_version, AIDF_VERSION, '<' ) ) {
     145        aidf_create_crawler_tables();
     146    }
     147}
     148add_action( 'admin_init', 'aidf_maybe_upgrade_db' );

  • ai-discovery-files/tags/1.3.0/includes/class-plugin.php

    r3472434 r3488445  
    3939     */
    4040    private function __construct() {
     41        AIDF_Crawler_Logger::init();
    4142        AIDF_Server::init();
    4243
    … …  
    126127            // Directory verification.
    127128            'verify_code'        => '',
     129
     130            // Crawler analytics.
     131            'crawler_logging_enabled' => false,
     132            'crawler_log_retention'   => '90',
     133            'crawler_enabled_bots'    => AIDF_Crawler_Registry::get_default_bot_keys(),
    128134        );
    129135

  • ai-discovery-files/tags/1.3.0/readme.txt

    r3487801 r3488445  
    55Tested up to: 6.9
    66Requires PHP: 8.0
    7 Stable tag: 1.2.1
     7Stable tag: 1.3.0
    88License: GPLv2 or later
    99License URI: https://www.gnu.org/licenses/gpl-2.0.html
    1010
    11 The only WordPress plugin built on the official AI Discovery Files Specification. Generates all 10 files. Built by 365i.
     11The only WordPress plugin that generates all 10 AI Discovery Files and tracks which AI bots visit your site. Built by 365i.
    1212
    1313== Description ==
    … …  
    6666* **Live preview** — see exactly what each file contains before enabling it
    6767* **Validation** — checks files against the specification and flags issues
     68* **AI Crawler Analytics** — see which AI bots visit your site, how often, and which pages they access
     69* **Discovery File Access tracking** — proof that AI bots are reading the files this plugin generates
     70* **robots.txt conflict detection** — warns when your robots.txt contradicts your AI visibility settings
    6871* **Conflict detection** — warns if physical files already exist at the same URLs
    6972* **Directory verification** — verify domain ownership for the [AI Visibility Directory](https://www.ai-visibility.org.uk/) without FTP
    … …  
    145148}, 10, 3 );`
    146149
     150= What is AI Crawler Analytics? =
     151
     152AI Crawler Analytics shows you which AI bots are visiting your site — GPTBot, ClaudeBot, PerplexityBot, Applebot, and 40+ others. You can see how often they visit, which pages they access, and whether any are being blocked by your robots.txt. It also shows which of your AI Discovery Files are being read by AI crawlers, giving you proof that the plugin is working.
     153
     154= Will the crawler analytics slow down my website? =
     155
     156No. When crawler logging is enabled, the plugin checks each request's user agent against a list of known AI bots. For non-bot requests (99.9% of traffic), this check takes less than a millisecond and involves no database queries. Bot hits are buffered in memory and written to the database once per request on shutdown. When logging is disabled (the default), the check is not registered at all — zero overhead.
     157
    147158= Will this slow down my website? =
    148159
    149 No. The plugin only runs when its specific URLs are requested (e.g., `/llms.txt`). It adds zero overhead to your normal page loads. Files are generated on-the-fly from cached settings data.
     160No. The plugin only runs when its specific URLs are requested (e.g., `/llms.txt`). It adds zero overhead to your normal page loads. Files are generated on-the-fly from cached settings data. The crawler analytics feature is disabled by default and adds negligible overhead when enabled.
    150161
    151162= What happens if I deactivate the plugin? =
    … …  
    167178
    168179== Changelog ==
     180
     181= 1.3.0 =
     182* New: AI Crawler Analytics — see which AI bots visit your site
     183* New: Dashboard showing bot visits, frequency, and pages accessed
     184* New: Discovery File Access panel — see which bots read your AI Discovery Files
     185* New: robots.txt conflict detection with actionable alerts
     186* New: Categorised bot registry with 43 AI crawlers across 8 categories
     187* New: Filterable activity log viewer with CSV export
     188* New: Bot detail drill-down with visit trends and page breakdown
     189* New: Two WordPress dashboard widgets (Crawler Activity + File Access)
     190* New: User controls — enable/disable logging, data retention, bot selection
    169191
    170192= 1.2.1 =
    … …  
    207229== Upgrade Notice ==
    208230
     231= 1.3.0 =
     232New: AI Crawler Analytics. See which AI bots visit your site, track Discovery File access, and detect robots.txt conflicts. Includes dashboard widgets, bot detail drill-downs, filterable log viewer, and CSV export.
     233
    209234= 1.1.0 =
    210235New: Domain verification for the AI Visibility Directory. Serve your verification file directly from WordPress — no FTP needed.

  • ai-discovery-files/tags/1.3.0/uninstall.php

    r3475304 r3488445  
    2828delete_option( 'aidf_settings_saved_at' );
    2929
     30// Drop crawler analytics tables.
     31global $wpdb;
     32$wpdb->query( "DROP TABLE IF EXISTS {$wpdb->prefix}aidf_crawler_log" );     // phpcs:ignore WordPress.DB.DirectDatabaseQuery
     33$wpdb->query( "DROP TABLE IF EXISTS {$wpdb->prefix}aidf_crawler_summary" ); // phpcs:ignore WordPress.DB.DirectDatabaseQuery
     34
     35// Remove crawler-specific options and transients.
     36delete_option( 'aidf_db_version' );
     37delete_transient( 'aidf_robots_txt_cache' );
     38wp_clear_scheduled_hook( 'aidf_rebuild_crawler_summary' );
     39wp_clear_scheduled_hook( 'aidf_prune_crawler_log' );
     40
    3041// Flush rewrite rules to remove our custom rules.
    3142flush_rewrite_rules();

  • ai-discovery-files/trunk/admin/class-admin.php

    r3482407 r3488445  
    3939        add_action( 'wp_ajax_aidf_dismiss_review', array( __CLASS__, 'ajax_dismiss_review' ) );
    4040        add_action( 'wp_ajax_aidf_search_pages', array( __CLASS__, 'ajax_search_pages' ) );
     41        add_action( 'wp_ajax_aidf_crawler_overview', array( __CLASS__, 'ajax_crawler_overview' ) );
     42        add_action( 'wp_ajax_aidf_crawler_bot_detail', array( __CLASS__, 'ajax_crawler_bot_detail' ) );
     43        add_action( 'wp_ajax_aidf_crawler_log_entries', array( __CLASS__, 'ajax_crawler_log_entries' ) );
     44        add_action( 'wp_ajax_aidf_crawler_export_csv', array( __CLASS__, 'ajax_crawler_export_csv' ) );
     45        add_action( 'wp_ajax_aidf_crawler_clear_log', array( __CLASS__, 'ajax_crawler_clear_log' ) );
     46        add_action( 'wp_ajax_aidf_crawler_conflicts', array( __CLASS__, 'ajax_crawler_conflicts' ) );
     47        add_action( 'wp_ajax_aidf_crawler_file_access', array( __CLASS__, 'ajax_crawler_file_access' ) );
     48        add_action( 'wp_ajax_aidf_crawler_save_settings', array( __CLASS__, 'ajax_crawler_save_settings' ) );
     49        add_action( 'wp_dashboard_setup', array( __CLASS__, 'register_dashboard_widgets' ) );
     50        add_action( 'admin_head-index.php', array( __CLASS__, 'dashboard_widget_styles' ) );
    4151        add_action( 'admin_notices', array( __CLASS__, 'maybe_show_welcome_notice' ) );
    4252        add_action( 'admin_notices', array( __CLASS__, 'maybe_show_conflict_notice' ) );
    … …  
    123133                    'noResults'       => __( 'No pages found', 'ai-discovery-files' ),
    124134                    'typeToSearch'    => __( 'Type to search…', 'ai-discovery-files' ),
     135                    'saving'          => __( 'Saving...', 'ai-discovery-files' ),
     136                    'saveSettings'    => __( 'Save Settings', 'ai-discovery-files' ),
     137                    'confirmClearLog' => __( 'This permanently deletes all crawler log data. This cannot be undone. Continue?', 'ai-discovery-files' ),
     138                    'noDataYet'       => __( 'No data for this period.', 'ai-discovery-files' ),
     139                    'justNow'         => __( 'Just now', 'ai-discovery-files' ),
     140                    /* translators: %d: number of minutes ago */
     141                    'mAgo'            => __( '%dm ago', 'ai-discovery-files' ),
     142                    /* translators: %d: number of hours ago */
     143                    'hAgo'            => __( '%dh ago', 'ai-discovery-files' ),
     144                    /* translators: %d: number of days ago */
     145                    'dAgo'            => __( '%dd ago', 'ai-discovery-files' ),
     146                    'loading'         => __( 'Loading...', 'ai-discovery-files' ),
    125147                ),
    126148            )
    127149        );
     150
     151        $current_tab = self::get_current_tab();
     152        if ( 'crawlers' === $current_tab ) {
     153            wp_enqueue_style( 'aidf-crawlers', AIDF_PLUGIN_URL . 'admin/css/crawlers.css', array( 'aidf-admin' ), AIDF_VERSION );
     154            wp_enqueue_script( 'aidf-crawlers', AIDF_PLUGIN_URL . 'admin/js/crawlers.js', array( 'jquery' ), AIDF_VERSION, true );
     155        }
    128156    }
    129157
    … …  
    287315        $tab = isset( $_GET['tab'] ) ? sanitize_text_field( wp_unslash( $_GET['tab'] ) ) : 'identity';
    288316
    289         $valid_tabs = array( 'identity', 'permissions', 'content', 'advanced', 'preview', 'status' );
     317        $valid_tabs = array( 'identity', 'permissions', 'content', 'advanced', 'preview', 'status', 'crawlers' );
    290318
    291319        if ( ! in_array( $tab, $valid_tabs, true ) ) {
    … …  
    309337            'preview'     => __( 'Preview', 'ai-discovery-files' ),
    310338            'status'      => __( 'Status', 'ai-discovery-files' ),
     339            'crawlers'    => __( 'AI Crawlers', 'ai-discovery-files' ),
    311340        );
    312341    }
    … …  
    658687        wp_send_json_success( $results );
    659688    }
     689
     690    /**
     691     * Register WordPress dashboard widgets for crawler analytics.
     692     *
     693     * @since 1.3.0
     694     */
     695    public static function register_dashboard_widgets() {
     696        $settings = AIDF_Plugin::get_settings();
     697        if ( empty( $settings['crawler_logging_enabled'] ) ) {
     698            return;
     699        }
     700
     701        wp_add_dashboard_widget(
     702            'aidf_crawler_activity',
     703            __( 'AI Crawler Activity', 'ai-discovery-files' ),
     704            array( __CLASS__, 'render_crawler_activity_widget' )
     705        );
     706
     707        wp_add_dashboard_widget(
     708            'aidf_file_access',
     709            __( 'AI Discovery File Access', 'ai-discovery-files' ),
     710            array( __CLASS__, 'render_file_access_widget' )
     711        );
     712    }
     713
     714    /**
     715     * Render the Crawler Activity dashboard widget.
     716     *
     717     * @since 1.3.0
     718     */
     719    public static function render_crawler_activity_widget() {
     720        include AIDF_PLUGIN_DIR . 'admin/views/partials/crawler-dashboard-widget.php';
     721    }
     722
     723    /**
     724     * Render the File Access dashboard widget.
     725     *
     726     * @since 1.3.0
     727     */
     728    public static function render_file_access_widget() {
     729        include AIDF_PLUGIN_DIR . 'admin/views/partials/crawler-file-access-widget.php';
     730    }
     731
     732    /**
     733     * Output inline CSS for dashboard widgets (only on the Dashboard screen).
     734     *
     735     * Gated on crawler_logging_enabled so styles are not injected when the
     736     * feature is disabled and the widgets are not registered.
     737     *
     738     * @since 1.3.0
     739     */
     740    public static function dashboard_widget_styles() {
     741        $settings = AIDF_Plugin::get_settings();
     742        if ( empty( $settings['crawler_logging_enabled'] ) ) {
     743            return;
     744        }
     745        ?>
     746        <style>
     747        /* AI Discovery Files — Dashboard Widget Styles */
     748        .aidf-widget-top-bots { color: #5a5d6b; font-size: 13px; }
     749        .aidf-widget-warning { color: #d97706; font-size: 13px; }
     750        .aidf-widget-warning .dashicons { font-size: 16px; width: 16px; height: 16px; vertical-align: text-bottom; }
     751        .aidf-widget-empty { color: #8b8fa3; font-style: italic; }
     752        .aidf-widget-link { text-align: right; margin-bottom: 0; }
     753        .aidf-widget-link a { color: #e77d15; text-decoration: none; font-weight: 500; }
     754        .aidf-widget-link a:hover { color: #d06e0d; }
     755        .aidf-widget-files { display: flex; flex-direction: column; gap: 6px; margin-bottom: 12px; }
     756        .aidf-widget-file-row { display: flex; align-items: center; gap: 8px; font-size: 12px; }
     757        .aidf-widget-file-name { font-family: ui-monospace, 'SF Mono', Menlo, Consolas, monospace; width: 110px; flex-shrink: 0; color: #1a1a2e; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
     758        .aidf-widget-file-bar { flex: 1; height: 8px; background: #e5e7eb; border-radius: 4px; overflow: hidden; }
     759        .aidf-widget-file-bar-fill { display: block; height: 100%; background: #e77d15; border-radius: 4px; transition: width 0.3s ease; }
     760        .aidf-widget-file-count { width: 30px; text-align: right; font-weight: 600; color: #1a1a2e; flex-shrink: 0; }
     761        .aidf-widget-file-bots { width: 90px; flex-shrink: 0; color: #5a5d6b; font-size: 11px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
     762        .aidf-widget-more { color: #8b8fa3; }
     763        .aidf-widget-empty-text { color: #8b8fa3; }
     764        .aidf-widget-summary { color: #5a5d6b; font-size: 13px; }
     765        .aidf-widget-summary strong { color: #1a1a2e; }
     766        </style>
     767        <?php
     768    }
     769
     770    /**
     771     * AJAX: Crawler overview data (summary + chart + breakdown).
     772     *
     773     * @since 1.3.0
     774     */
     775    public static function ajax_crawler_overview() {
     776        check_ajax_referer( 'aidf_admin_nonce', 'nonce' );
     777        if ( ! current_user_can( 'manage_options' ) ) {
     778            wp_send_json_error( __( 'Permission denied.', 'ai-discovery-files' ) );
     779        }
     780
     781        $days = isset( $_GET['days'] ) ? absint( $_GET['days'] ) : 30;
     782        if ( ! in_array( $days, array( 7, 30, 90 ), true ) ) {
     783            $days = 30;
     784        }
     785
     786        wp_send_json_success( array(
     787            'summary'   => AIDF_Crawler_Analytics::get_summary( $days ),
     788            'chart'     => AIDF_Crawler_Analytics::get_chart_data( $days ),
     789            'breakdown' => AIDF_Crawler_Analytics::get_bot_breakdown( $days ),
     790            'days'      => $days,
     791        ) );
     792    }
     793
     794    /**
     795     * AJAX: Crawler bot detail.
     796     *
     797     * Returns detailed analytics for a single bot: daily trend, status code
     798     * breakdown, pages visited, discovery file access, and first/last seen.
     799     *
     800     * @since 1.3.0
     801     */
     802    public static function ajax_crawler_bot_detail() {
     803        check_ajax_referer( 'aidf_admin_nonce', 'nonce' );
     804        if ( ! current_user_can( 'manage_options' ) ) {
     805            wp_send_json_error( __( 'Permission denied.', 'ai-discovery-files' ) );
     806        }
     807
     808        $bot_name = isset( $_GET['bot_name'] ) ? sanitize_text_field( wp_unslash( $_GET['bot_name'] ) ) : '';
     809        if ( empty( $bot_name ) ) {
     810            wp_send_json_error( __( 'Bot name required.', 'ai-discovery-files' ) );
     811        }
     812
     813        $days = isset( $_GET['days'] ) ? absint( $_GET['days'] ) : 30;
     814        if ( ! in_array( $days, array( 7, 30, 90 ), true ) ) {
     815            $days = 30;
     816        }
     817
     818        wp_send_json_success( AIDF_Crawler_Analytics::get_bot_detail( $bot_name, $days ) );
     819    }
     820
     821    /**
     822     * AJAX: Crawler log entries.
     823     *
     824     * Returns a paginated, filtered slice of the raw crawler log.
     825     * Supported filters: bot_name, date_from, date_to, status_code, url_path.
     826     *
     827     * @since 1.3.0
     828     */
     829    public static function ajax_crawler_log_entries() {
     830        check_ajax_referer( 'aidf_admin_nonce', 'nonce' );
     831        if ( ! current_user_can( 'manage_options' ) ) {
     832            wp_send_json_error( __( 'Permission denied.', 'ai-discovery-files' ) );
     833        }
     834
     835        $args = array(
     836            'bot_name'    => isset( $_GET['bot_name'] ) ? sanitize_text_field( wp_unslash( $_GET['bot_name'] ) ) : '',
     837            'date_from'   => isset( $_GET['date_from'] ) ? sanitize_text_field( wp_unslash( $_GET['date_from'] ) ) : '',
     838            'date_to'     => isset( $_GET['date_to'] ) ? sanitize_text_field( wp_unslash( $_GET['date_to'] ) ) : '',
     839            'status_code' => isset( $_GET['status_code'] ) ? sanitize_text_field( wp_unslash( $_GET['status_code'] ) ) : '',
     840            'url_path'    => isset( $_GET['url_path'] ) ? sanitize_text_field( wp_unslash( $_GET['url_path'] ) ) : '',
     841            'page'        => isset( $_GET['paged'] ) ? absint( $_GET['paged'] ) : 1,
     842            'per_page'    => 50,
     843            'orderby'     => isset( $_GET['orderby'] ) ? sanitize_text_field( wp_unslash( $_GET['orderby'] ) ) : 'created_at',
     844            'order'       => isset( $_GET['order'] ) ? sanitize_text_field( wp_unslash( $_GET['order'] ) ) : 'DESC',
     845        );
     846
     847        wp_send_json_success( AIDF_Crawler_Analytics::get_log_entries( $args ) );
     848    }
     849
     850    /**
     851     * AJAX: Export crawler log as CSV.
     852     *
     853     * Streams a UTF-8 CSV file with all matching log entries and exits.
     854     * Filter parameters mirror those of ajax_crawler_log_entries.
     855     *
     856     * @since 1.3.0
     857     */
     858    public static function ajax_crawler_export_csv() {
     859        check_ajax_referer( 'aidf_admin_nonce', 'nonce' );
     860        if ( ! current_user_can( 'manage_options' ) ) {
     861            wp_die( esc_html__( 'Permission denied.', 'ai-discovery-files' ) );
     862        }
     863
     864        $args = array(
     865            'bot_name'    => isset( $_GET['bot_name'] ) ? sanitize_text_field( wp_unslash( $_GET['bot_name'] ) ) : '',
     866            'date_from'   => isset( $_GET['date_from'] ) ? sanitize_text_field( wp_unslash( $_GET['date_from'] ) ) : '',
     867            'date_to'     => isset( $_GET['date_to'] ) ? sanitize_text_field( wp_unslash( $_GET['date_to'] ) ) : '',
     868            'status_code' => isset( $_GET['status_code'] ) ? sanitize_text_field( wp_unslash( $_GET['status_code'] ) ) : '',
     869            'url_path'    => isset( $_GET['url_path'] ) ? sanitize_text_field( wp_unslash( $_GET['url_path'] ) ) : '',
     870        );
     871
     872        AIDF_Crawler_Analytics::export_csv( $args );
     873    }
     874
     875    /**
     876     * AJAX: Clear crawler log.
     877     *
     878     * @since 1.3.0
     879     */
     880    public static function ajax_crawler_clear_log() {
     881        check_ajax_referer( 'aidf_admin_nonce', 'nonce' );
     882        if ( ! current_user_can( 'manage_options' ) ) {
     883            wp_send_json_error( __( 'Permission denied.', 'ai-discovery-files' ) );
     884        }
     885
     886        global $wpdb;
     887
     888        // Use DELETE instead of TRUNCATE for shared hosting compatibility
     889        // (TRUNCATE requires DROP privilege which many hosts don't grant).
     890        // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching
     891        $wpdb->query( "DELETE FROM {$wpdb->prefix}aidf_crawler_log" );
     892        // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching
     893        $wpdb->query( "DELETE FROM {$wpdb->prefix}aidf_crawler_summary" );
     894
     895        wp_send_json_success( array(
     896            'message' => __( 'Log data cleared.', 'ai-discovery-files' ),
     897        ) );
     898    }
     899
     900    /**
     901     * AJAX: Crawler conflicts data.
     902     *
     903     * @since 1.3.0
     904     */
     905    public static function ajax_crawler_conflicts() {
     906        check_ajax_referer( 'aidf_admin_nonce', 'nonce' );
     907        if ( ! current_user_can( 'manage_options' ) ) {
     908            wp_send_json_error( __( 'Permission denied.', 'ai-discovery-files' ) );
     909        }
     910
     911        $conflicts = AIDF_Crawler_Conflicts::get_conflicts();
     912
     913        // Sanitize messages for safe DOM insertion in JavaScript.
     914        foreach ( $conflicts as &$alert ) {
     915            $alert['message'] = wp_kses_post( $alert['message'] );
     916        }
     917        unset( $alert );
     918
     919        wp_send_json_success( array(
     920            'conflicts' => $conflicts,
     921        ) );
     922    }
     923
     924    /**
     925     * AJAX: Crawler file access data.
     926     *
     927     * @since 1.3.0
     928     */
     929    public static function ajax_crawler_file_access() {
     930        check_ajax_referer( 'aidf_admin_nonce', 'nonce' );
     931        if ( ! current_user_can( 'manage_options' ) ) {
     932            wp_send_json_error( __( 'Permission denied.', 'ai-discovery-files' ) );
     933        }
     934
     935        $days = isset( $_GET['days'] ) ? absint( $_GET['days'] ) : 30;
     936        if ( ! in_array( $days, array( 7, 30, 90 ), true ) ) {
     937            $days = 30;
     938        }
     939
     940        wp_send_json_success( array(
     941            'file_access' => AIDF_Crawler_Analytics::get_file_access( $days ),
     942            'days'        => $days,
     943        ) );
     944    }
     945
     946    /**
     947     * AJAX: Save crawler settings.
     948     *
     949     * @since 1.3.0
     950     */
     951    public static function ajax_crawler_save_settings() {
     952        check_ajax_referer( 'aidf_admin_nonce', 'nonce' );
     953        if ( ! current_user_can( 'manage_options' ) ) {
     954            wp_send_json_error( __( 'Permission denied.', 'ai-discovery-files' ) );
     955        }
     956
     957        $settings    = AIDF_Plugin::get_settings();
     958        $was_enabled = ! empty( $settings['crawler_logging_enabled'] );
     959
     960        // Logging toggle.
     961        $settings['crawler_logging_enabled'] = ! empty( $_POST['crawler_logging_enabled'] );
     962
     963        // Retention period.
     964        $valid_retention = array( '30', '60', '90', '180', '365' );
     965        $retention       = isset( $_POST['crawler_log_retention'] )
     966            ? sanitize_text_field( wp_unslash( $_POST['crawler_log_retention'] ) )
     967            : '90';
     968        $settings['crawler_log_retention'] = in_array( $retention, $valid_retention, true ) ? $retention : '90';
     969
     970        // Enabled bots.
     971        $enabled_bots = array();
     972        $raw_bots = isset( $_POST['crawler_enabled_bots'] ) ? array_map( 'sanitize_text_field', wp_unslash( (array) $_POST['crawler_enabled_bots'] ) ) : array();
     973        if ( ! empty( $raw_bots ) ) {
     974            $all_bot_keys = array_keys( AIDF_Crawler_Registry::get_bots() );
     975            foreach ( $raw_bots as $key ) {
     976                if ( in_array( $key, $all_bot_keys, true ) ) {
     977                    $enabled_bots[] = $key;
     978                }
     979            }
     980        }
     981        $settings['crawler_enabled_bots'] = $enabled_bots;
     982
     983        update_option( 'aidf_settings', $settings );
     984
     985        // Manage cron based on toggle change.
     986        $is_enabled = ! empty( $settings['crawler_logging_enabled'] );
     987        if ( $is_enabled && ! $was_enabled ) {
     988            AIDF_Crawler_Logger::schedule_cron();
     989        } elseif ( ! $is_enabled && $was_enabled ) {
     990            AIDF_Crawler_Logger::clear_cron();
     991        }
     992
     993        wp_send_json_success( array(
     994            'message' => __( 'Settings saved.', 'ai-discovery-files' ),
     995        ) );
     996    }
    660997}

  • ai-discovery-files/trunk/admin/class-settings.php

    r3475304 r3488445  
    278278        }
    279279
     280        // Preserve crawler analytics settings.
     281        // Read from $input first (AJAX save passes crawler keys via update_option),
     282        // fall back to $fallback (non-crawler tab saves don't include these keys).
     283        $clean['crawler_logging_enabled'] = isset( $input['crawler_logging_enabled'] )
     284            ? (bool) $input['crawler_logging_enabled']
     285            : ( isset( $fallback['crawler_logging_enabled'] ) ? (bool) $fallback['crawler_logging_enabled'] : false );
     286        $clean['crawler_log_retention'] = isset( $input['crawler_log_retention'] )
     287            ? sanitize_text_field( $input['crawler_log_retention'] )
     288            : ( isset( $fallback['crawler_log_retention'] ) ? $fallback['crawler_log_retention'] : '90' );
     289        $clean['crawler_enabled_bots'] = isset( $input['crawler_enabled_bots'] ) && is_array( $input['crawler_enabled_bots'] )
     290            ? array_map( 'sanitize_text_field', $input['crawler_enabled_bots'] )
     291            : ( isset( $fallback['crawler_enabled_bots'] ) ? $fallback['crawler_enabled_bots'] : array() );
     292
    280293        // Flush rewrite rules since active files may have changed.
    281294        AIDF_Server::register_rewrite_rules();

  • ai-discovery-files/trunk/admin/views/settings-page.php

    r3475304 r3488445  
    5151            'preview'     => 'dashicons-media-code',
    5252            'status'      => 'dashicons-yes-alt',
     53            'crawlers'    => 'dashicons-chart-bar',
    5354        );
    5455
    … …  
    7778        <?php include AIDF_PLUGIN_DIR . 'admin/views/partials/directory-cta.php'; ?>
    7879
    79         <?php if ( in_array( $current_tab, array( 'preview', 'status' ), true ) ) : ?>
     80        <?php if ( in_array( $current_tab, array( 'preview', 'status', 'crawlers' ), true ) ) : ?>
    8081            <?php include AIDF_PLUGIN_DIR . 'admin/views/tab-' . $current_tab . '.php'; ?>
    8182        <?php else : ?>
    … …  
    8586
    8687                <!-- Persist active_files and spec_attribution across all tabs -->
    87                 <?php if ( 'status' !== $current_tab ) : ?>
     88                <?php if ( ! in_array( $current_tab, array( 'status', 'crawlers' ), true ) ) : ?>
    8889                    <?php
    8990                    $active_files = isset( $settings['active_files'] ) ? (array) $settings['active_files'] : array();

  • ai-discovery-files/trunk/ai-discovery-files.php

    r3487801 r3488445  
    33 * Plugin Name:       AI Discovery Files – llms.txt & AI Visibility
    44 * Plugin URI:        https://www.ai-visibility.org.uk/wordpress-plugin/ai-discovery-files/
    5  * Description:       The only WordPress plugin built on the official AI Discovery Files Specification. Generates all 10 files. Built by 365i.
    6  * Version:           1.2.1
     5 * Description:       The only WordPress plugin that generates all 10 AI Discovery Files and tracks which AI bots visit your site. Built by 365i.
     6 * Version:           1.3.0
    77 * Requires at least: 6.2
    88 * Requires PHP:      8.0
    … …  
    2424 * Plugin constants.
    2525 */
    26 define( 'AIDF_VERSION', '1.2.1' );
     26define( 'AIDF_VERSION', '1.3.0' );
    2727define( 'AIDF_PLUGIN_FILE', __FILE__ );
    2828define( 'AIDF_PLUGIN_DIR', plugin_dir_path( __FILE__ ) );
    … …  
    3838require_once AIDF_PLUGIN_DIR . 'includes/class-server.php';
    3939require_once AIDF_PLUGIN_DIR . 'includes/class-validator.php';
     40require_once AIDF_PLUGIN_DIR . 'includes/class-crawler-registry.php';
     41require_once AIDF_PLUGIN_DIR . 'includes/class-crawler-logger.php';
     42require_once AIDF_PLUGIN_DIR . 'includes/class-crawler-analytics.php';
     43require_once AIDF_PLUGIN_DIR . 'includes/class-crawler-conflicts.php';
     44
     45/**
     46 * Create or upgrade the crawler analytics database tables.
     47 *
     48 * Uses dbDelta() so it is safe to call on both fresh installs and upgrades.
     49 * Stores the installed DB version in the aidf_db_version option.
     50 */
     51function aidf_create_crawler_tables() {
     52    global $wpdb;
     53
     54    $charset_collate = $wpdb->get_charset_collate();
     55
     56    $sql = "CREATE TABLE {$wpdb->prefix}aidf_crawler_log (
     57  id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
     58  bot_name varchar(50) NOT NULL,
     59  bot_ua varchar(500) NOT NULL DEFAULT '',
     60  url_path varchar(500) NOT NULL DEFAULT '',
     61  status_code smallint(5) unsigned NOT NULL DEFAULT 200,
     62  created_at datetime NOT NULL,
     63  PRIMARY KEY  (id),
     64  KEY bot_date (bot_name, created_at),
     65  KEY date_idx (created_at)
     66) $charset_collate;
     67
     68CREATE TABLE {$wpdb->prefix}aidf_crawler_summary (
     69  id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
     70  bot_name varchar(50) NOT NULL,
     71  log_date date NOT NULL,
     72  hit_count int(10) unsigned NOT NULL DEFAULT 0,
     73  top_page varchar(500) NOT NULL DEFAULT '',
     74  status_200 int(10) unsigned NOT NULL DEFAULT 0,
     75  status_403 int(10) unsigned NOT NULL DEFAULT 0,
     76  status_other int(10) unsigned NOT NULL DEFAULT 0,
     77  PRIMARY KEY  (id),
     78  UNIQUE KEY bot_day (bot_name, log_date),
     79  KEY date_idx (log_date)
     80) $charset_collate;";
     81
     82    require_once ABSPATH . 'wp-admin/includes/upgrade.php';
     83    dbDelta( $sql );
     84
     85    update_option( 'aidf_db_version', AIDF_VERSION );
     86}
    4087
    4188/**
    … …  
    55102
    56103    set_transient( 'aidf_activation_redirect', true, 30 );
     104
     105    aidf_create_crawler_tables();
    57106}
    58107register_activation_hook( __FILE__, 'aidf_activate' );
    … …  
    63112function aidf_deactivate() {
    64113    flush_rewrite_rules();
     114    AIDF_Crawler_Logger::clear_cron();
    65115}
    66116register_deactivation_hook( __FILE__, 'aidf_deactivate' );
    … …  
    73123}
    74124add_action( 'plugins_loaded', 'aidf_init' );
     125
     126/**
     127 * Cron: prune old crawler log entries.
     128 */
     129add_action( 'aidf_prune_crawler_log', array( 'AIDF_Crawler_Logger', 'prune_log' ) );
     130
     131/**
     132 * Cron: rebuild crawler summary table from raw log data.
     133 */
     134add_action( 'aidf_rebuild_crawler_summary', array( 'AIDF_Crawler_Analytics', 'rebuild_summary' ) );
     135
     136/**
     137 * Run DB migrations when the plugin version has advanced.
     138 *
     139 * Hooked to admin_init so it runs on every admin page load, but only
     140 * performs work when the stored version is behind the current version.
     141 */
     142function aidf_maybe_upgrade_db() {
     143    $installed_version = get_option( 'aidf_db_version', '0' );
     144    if ( version_compare( $installed_version, AIDF_VERSION, '<' ) ) {
     145        aidf_create_crawler_tables();
     146    }
     147}
     148add_action( 'admin_init', 'aidf_maybe_upgrade_db' );

  • ai-discovery-files/trunk/includes/class-plugin.php

    r3472434 r3488445  
    3939     */
    4040    private function __construct() {
     41        AIDF_Crawler_Logger::init();
    4142        AIDF_Server::init();
    4243
    … …  
    126127            // Directory verification.
    127128            'verify_code'        => '',
     129
     130            // Crawler analytics.
     131            'crawler_logging_enabled' => false,
     132            'crawler_log_retention'   => '90',
     133            'crawler_enabled_bots'    => AIDF_Crawler_Registry::get_default_bot_keys(),
    128134        );
    129135

  • ai-discovery-files/trunk/readme.txt

    r3487801 r3488445  
    55Tested up to: 6.9
    66Requires PHP: 8.0
    7 Stable tag: 1.2.1
     7Stable tag: 1.3.0
    88License: GPLv2 or later
    99License URI: https://www.gnu.org/licenses/gpl-2.0.html
    1010
    11 The only WordPress plugin built on the official AI Discovery Files Specification. Generates all 10 files. Built by 365i.
     11The only WordPress plugin that generates all 10 AI Discovery Files and tracks which AI bots visit your site. Built by 365i.
    1212
    1313== Description ==
    … …  
    6666* **Live preview** — see exactly what each file contains before enabling it
    6767* **Validation** — checks files against the specification and flags issues
     68* **AI Crawler Analytics** — see which AI bots visit your site, how often, and which pages they access
     69* **Discovery File Access tracking** — proof that AI bots are reading the files this plugin generates
     70* **robots.txt conflict detection** — warns when your robots.txt contradicts your AI visibility settings
    6871* **Conflict detection** — warns if physical files already exist at the same URLs
    6972* **Directory verification** — verify domain ownership for the [AI Visibility Directory](https://www.ai-visibility.org.uk/) without FTP
    … …  
    145148}, 10, 3 );`
    146149
     150= What is AI Crawler Analytics? =
     151
     152AI Crawler Analytics shows you which AI bots are visiting your site — GPTBot, ClaudeBot, PerplexityBot, Applebot, and 40+ others. You can see how often they visit, which pages they access, and whether any are being blocked by your robots.txt. It also shows which of your AI Discovery Files are being read by AI crawlers, giving you proof that the plugin is working.
     153
     154= Will the crawler analytics slow down my website? =
     155
     156No. When crawler logging is enabled, the plugin checks each request's user agent against a list of known AI bots. For non-bot requests (99.9% of traffic), this check takes less than a millisecond and involves no database queries. Bot hits are buffered in memory and written to the database once per request on shutdown. When logging is disabled (the default), the check is not registered at all — zero overhead.
     157
    147158= Will this slow down my website? =
    148159
    149 No. The plugin only runs when its specific URLs are requested (e.g., `/llms.txt`). It adds zero overhead to your normal page loads. Files are generated on-the-fly from cached settings data.
     160No. The plugin only runs when its specific URLs are requested (e.g., `/llms.txt`). It adds zero overhead to your normal page loads. Files are generated on-the-fly from cached settings data. The crawler analytics feature is disabled by default and adds negligible overhead when enabled.
    150161
    151162= What happens if I deactivate the plugin? =
    … …  
    167178
    168179== Changelog ==
     180
     181= 1.3.0 =
     182* New: AI Crawler Analytics — see which AI bots visit your site
     183* New: Dashboard showing bot visits, frequency, and pages accessed
     184* New: Discovery File Access panel — see which bots read your AI Discovery Files
     185* New: robots.txt conflict detection with actionable alerts
     186* New: Categorised bot registry with 43 AI crawlers across 8 categories
     187* New: Filterable activity log viewer with CSV export
     188* New: Bot detail drill-down with visit trends and page breakdown
     189* New: Two WordPress dashboard widgets (Crawler Activity + File Access)
     190* New: User controls — enable/disable logging, data retention, bot selection
    169191
    170192= 1.2.1 =
    … …  
    207229== Upgrade Notice ==
    208230
     231= 1.3.0 =
     232New: AI Crawler Analytics. See which AI bots visit your site, track Discovery File access, and detect robots.txt conflicts. Includes dashboard widgets, bot detail drill-downs, filterable log viewer, and CSV export.
     233
    209234= 1.1.0 =
    210235New: Domain verification for the AI Visibility Directory. Serve your verification file directly from WordPress — no FTP needed.

  • ai-discovery-files/trunk/uninstall.php

    r3475304 r3488445  
    2828delete_option( 'aidf_settings_saved_at' );
    2929
     30// Drop crawler analytics tables.
     31global $wpdb;
     32$wpdb->query( "DROP TABLE IF EXISTS {$wpdb->prefix}aidf_crawler_log" );     // phpcs:ignore WordPress.DB.DirectDatabaseQuery
     33$wpdb->query( "DROP TABLE IF EXISTS {$wpdb->prefix}aidf_crawler_summary" ); // phpcs:ignore WordPress.DB.DirectDatabaseQuery
     34
     35// Remove crawler-specific options and transients.
     36delete_option( 'aidf_db_version' );
     37delete_transient( 'aidf_robots_txt_cache' );
     38wp_clear_scheduled_hook( 'aidf_rebuild_crawler_summary' );
     39wp_clear_scheduled_hook( 'aidf_prune_crawler_log' );
     40
    3041// Flush rewrite rules to remove our custom rules.
    3142flush_rewrite_rules();
Note: See TracChangeset for help on using the changeset viewer.