ghost.db stores timestamps in ms, not seconds. Fixed the ghost importer. by pwntr · Pull Request #396 · jekyll/jekyll-import

pwntr · 2019-01-25T16:21:44Z

fixed it in the according importer by dividing the timestamp value by 1000

…hem in ms

lib/jekyll-import/importers/ghost.rb

Thx for the style and grammar correction, much appreciated :)! Co-Authored-By: pwntr <pwntr@users.noreply.github.com>

ashmaroli · 2019-01-30T09:33:37Z

@pwntr Do you have a link to some documentation where we can verify your claim that the database stores time in milliseconds..?

pwntr · 2019-01-30T10:23:28Z

@pwntr Do you have a link to some documentation where we can verify your claim that the database stores time in milliseconds..?

I based this on my own observations with my ghost.db backup when looking at the schema in a sqlite viewer. After your comment I crawled around the web and found more evidence: TryGhost/Ghost#2689 (comment)

Checking the relevant code of the current master branch of Ghost reveals the parts where javascript Date.now(); is being called. This always results in milliseconds for the timestamp:

        './core/server/apps/private-blogging/lib/middleware.js' :  const salt = Date.now().toString();
                         './core/server/lib/security/tokens.js' :  if (parts.expires < Date.now()) {
                     './core/server/api/v0.1/authentication.js' :  expires: Date.now() + constants.ONE_DAY_MS,
                     './core/server/api/v0.1/authentication.js' :  if (invite.get('expires') < Date.now()) {
                               './core/server/models/invite.js' :  data.expires = Date.now() + constants.ONE_WEEK_MS;
                           './core/server/models/base/token.js' :  .query('where', 'expires', '<', Date.now())
                                 './core/server/models/post.js' :  created_at_ts: Date.now() - 1
                                 './core/server/models/post.js' :  created_at_ts: Date.now()
                                 './core/server/models/post.js' :  created_at_ts: Date.now()
'./core/server/web/shared/middlewares/update-user-last-seen.js' :  if (Date.now() - req.user.get('last_seen') < constants.ONE_HOUR_MS) {
          './core/server/web/shared/middlewares/log-request.js' :  const startTime = Date.now(),
          './core/server/web/shared/middlewares/log-request.js' :  res.responseTime = (Date.now() - startTime) + 'ms';
                         './core/server/data/meta/asset_url.js' :  config.set('assetHash', (crypto.createHash('md5').update(packageInfo.version + Date.now()).digest('hex')).substring(0, 10));
             './core/server/data/xml/sitemap/base-generator.js' :  this.lastModified = Date.now();
                          './core/server/services/url/utils.js' :  publishedAtMoment = moment.tz(resource.published_at || Date.now(), settingsCache.get('active_timezone')),
               './core/server/services/auth/auth-strategies.js' :  if (token.expires > Date.now()) {
               './core/server/services/auth/auth-strategies.js' :  if (invite.get('expires') < Date.now()) {
                         './core/server/services/auth/oauth.js' :  if (token.expires <= Date.now()) {
                         './core/server/services/auth/utils.js' :  expires: Date.now() + constants.FIVE_MINUTES_MS
                         './core/server/services/auth/utils.js' :  accessExpires = Date.now() + constants.ONE_MONTH_MS,
                         './core/server/services/auth/utils.js' :  refreshExpires = Date.now() + constants.SIX_MONTH_MS,
                   './core/server/services/webhooks/trigger.js' :  const triggeredAt = Date.now();
    './core/test/unit/apps/private-blogging/middleware_spec.js' :  var salt = Date.now().toString();
    './core/test/unit/apps/private-blogging/middleware_spec.js' :  salt: Date.now().toString()
    './core/test/unit/apps/private-blogging/middleware_spec.js' :  var salt = Date.now().toString();
    './core/test/unit/apps/private-blogging/middleware_spec.js' :  salt: Date.now().toString()
    './core/test/unit/apps/private-blogging/middleware_spec.js' :  var salt = Date.now().toString();
    './core/test/unit/apps/private-blogging/middleware_spec.js' :  var salt = Date.now().toString();
    './core/test/unit/apps/private-blogging/middleware_spec.js' :  var salt = Date.now().toString();
    './core/test/unit/apps/private-blogging/middleware_spec.js' :  var salt = Date.now().toString();
    './core/test/unit/apps/private-blogging/middleware_spec.js' :  var salt = Date.now().toString();
    './core/test/unit/apps/private-blogging/middleware_spec.js' :  var salt = Date.now().toString();
    './core/test/unit/apps/private-blogging/middleware_spec.js' :  var salt = Date.now().toString();
                 './core/test/unit/lib/security/tokens_spec.js' :  var expires = Date.now() + 60 * 1000,
                 './core/test/unit/lib/security/tokens_spec.js' :  var expires = Date.now() + 60 * 1000,
                 './core/test/unit/lib/security/tokens_spec.js' :  var expires = Date.now() + 60 * 1000,
                 './core/test/unit/lib/security/tokens_spec.js' :  var expires = Date.now() + 60 * 1000,
                 './core/test/unit/lib/security/tokens_spec.js' :  var expires = Date.now() + 60 * 1000,
                 './core/test/unit/lib/security/tokens_spec.js' :  var expires = Date.now() + 60 * 1000,
'./core/test/unit/web/middleware/update-user-last-seen_spec.js' :  const fakeLastSeen = new Date(Date.now() - constants.ONE_HOURS_MS);
'./core/test/unit/web/middleware/update-user-last-seen_spec.js' :  const fakeLastSeen = new Date(Date.now() - constants.ONE_HOURS_MS);
                 './core/test/unit/services/auth/oauth_spec.js' :  expires_in: Date.now() + 1000
                 './core/test/unit/services/auth/oauth_spec.js' :  expires: Date.now() + 3600
                 './core/test/unit/services/auth/oauth_spec.js' :  expires_in: Date.now() + 1000
                 './core/test/unit/services/auth/oauth_spec.js' :  expires: Date.now() - 3600
                 './core/test/unit/services/auth/oauth_spec.js' :  expires: Date.now() + 3600
         './core/test/unit/services/auth/api-key/admin_spec.js' :  iat: Math.floor(Date.now() / 1000) - 6 * 60
         './core/test/unit/services/auth/api-key/admin_spec.js' :  iat: Math.floor(Date.now() / 1000) - 6 * 60
       './core/test/unit/services/auth/auth-strategies_spec.js' :  expires: Date.now() + constants.ONE_DAY_MS
       './core/test/unit/services/auth/auth-strategies_spec.js' :  expires: Date.now() - constants.ONE_DAY_MS
       './core/test/unit/services/auth/auth-strategies_spec.js' :  expires: Date.now() - 1000
       './core/test/unit/services/auth/auth-strategies_spec.js' :  expires: Date.now() + 2000,
                 './core/test/utils/fixtures/data-generator.js' :  newObj.slug = 'slug_' + Date.now() 
                 './core/test/utils/fixtures/data-generator.js' :  expires: Date.now() + constants.ONE_DAY_MS
                 './core/test/utils/fixtures/data-generator.js' :  expires: Date.now() + (60 * 1000)
       './core/test/regression/api/v0.1/authentication_spec.js' :  moment(Date.now() + constants.SIX_MONTH_MS)
       './core/test/regression/api/v0.1/authentication_spec.js' :  expires: Date.now() + (1000 * 60)
                                                   './index.js' :  var startTime = Date.now()
                                                   './index.js' :  common.logging.info('Ghost boot', (Date.now() - startTime) / 1000 + 's')

And for reference, the db schema of ghost itself: https://github.com/TryGhost/Ghost/blob/master/core/server/data/schema/schema.js

sqlite itself is agnostic when it comes to those timestamps, as ghost just stores them as strings.

ashmaroli · 2019-01-30T11:11:03Z

@pwntr Thank you for digging the info out.
Looking at it, I'm now confused because Ruby calculates time based on milliseconds passed from EPOCH as well and since our Ghost Importer doesn't have code that depends on time to be in seconds, what exactly could be the issue you're facing.... 😕

pwntr · 2019-01-30T12:14:22Z

@ashmaroli I believe Time.at expects seconds or fractions of seconds (with . as a separator) as input by default, at least that's what I found here. Entries in my ghost.db are milliseconds, hence the necessary conversion to match the expectation:

I'm a bit confused about the signature description of Time.at. as referenced in the link above: at(seconds, milliseconds, :millisecond) → time. Would that allow adding :millisecond to have the function evaluate it as such, or how would one go about it?

Either way, the input needs to be adjusted for Time.at. Dividing by 1000 might just be the quickest way.

ashmaroli · 2019-01-30T12:40:54Z

@pwntr Guess I was wrong.
From the same documentation, method to_i returns "the integer number of seconds since Epoch".
So Ruby calculates Time in terms of seconds passed from Epoch, while JavaScript does so in terms of milliseconds passed.

Sad, we don't have tests for all importers in this project.

pwntr · 2019-01-30T13:21:50Z

@ashmaroli Thx for the approval, glad we figured it out :)!

mattr- · 2019-01-30T14:31:55Z

@jekyllbot: merge +bug

mattr- · 2019-01-30T14:32:19Z

@pwntr Thanks for the contribution! We greatly appreciate it! 🎊

dividing the delivered timestamps by 1000 since the ghost.db stores t…

0f36ac9

…hem in ms

ashmaroli reviewed Jan 25, 2019

View reviewed changes

lib/jekyll-import/importers/ghost.rb Outdated Show resolved Hide resolved

Update lib/jekyll-import/importers/ghost.rb

f81055f

Thx for the style and grammar correction, much appreciated :)! Co-Authored-By: pwntr <pwntr@users.noreply.github.com>

ashmaroli requested a review from a team January 30, 2019 11:11

ashmaroli approved these changes Jan 30, 2019

View reviewed changes

jekyllbot merged commit b6fdf90 into jekyll:master Jan 30, 2019

jekyllbot added bug fix labels Jan 30, 2019

jekyllbot added a commit that referenced this pull request Jan 30, 2019

Update history to reflect merge of #396 [ci skip]

6e523de

jekyll locked and limited conversation to collaborators Jan 30, 2020

jekyllbot added the frozen-due-to-age label Jan 30, 2020

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

ghost.db stores timestamps in ms, not seconds. Fixed the ghost importer.#396

ghost.db stores timestamps in ms, not seconds. Fixed the ghost importer.#396
jekyllbot merged 2 commits intojekyll:masterfrom
pwntr:master

pwntr commented Jan 25, 2019

Uh oh!

Uh oh!

ashmaroli commented Jan 30, 2019

Uh oh!

pwntr commented Jan 30, 2019 •

edited by ashmaroli

Loading

Uh oh!

ashmaroli commented Jan 30, 2019

Uh oh!

pwntr commented Jan 30, 2019

Uh oh!

ashmaroli commented Jan 30, 2019

Uh oh!

pwntr commented Jan 30, 2019

Uh oh!

mattr- commented Jan 30, 2019

Uh oh!

mattr- commented Jan 30, 2019

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

4 participants

Conversation

pwntr commented Jan 25, 2019

Uh oh!

Uh oh!

ashmaroli commented Jan 30, 2019

Uh oh!

pwntr commented Jan 30, 2019 • edited by ashmaroli Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

ashmaroli commented Jan 30, 2019

Uh oh!

pwntr commented Jan 30, 2019

Uh oh!

ashmaroli commented Jan 30, 2019

Uh oh!

pwntr commented Jan 30, 2019

Uh oh!

mattr- commented Jan 30, 2019

Uh oh!

mattr- commented Jan 30, 2019

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

4 participants

pwntr commented Jan 30, 2019 •

edited by ashmaroli

Loading