course_link_validator.rb

require 'nokogiri'

class CourseLinkValidator

  TAG = "link_validation"

  # retrieves the validation job
  def self.current_progress(course)
    Progress.where(:tag => TAG, :context_type => 'Course', :context_id => course.id).last
  end

  # creates a new validation job
  def self.queue_course(course)
    progress = current_progress(course)
    return progress if progress && progress.pending?

    progress ||= Progress.new(:tag => TAG, :context => course)
    progress.reset!
    progress.process_job(self, :process)
    progress
  end

  def self.process(progress)
    validator = self.new(progress.context)
    validator.check_course(progress)
    progress.set_results({:issues => validator.issues, :completed_at => Time.now.utc})
  rescue
    report_id = Canvas::Errors.capture_exception(:course_link_validation, $ERROR_INFO)[:error_report]
    progress.workflow_state = 'failed'
    progress.set_results({error_report_id: report_id, completed_at: Time.now.utc})
  end

  attr_accessor :course, :domain_regex, :issues, :visited_urls

  def initialize(course)
    self.course = course
    domain = course.root_account.domain
    self.domain_regex = %r{\w+:?\/\/#{domain}\/} if domain
    self.issues = []
    self.visited_urls = {}
  end

  # ****************************************************************
  # this is where the magic happens
  def check_course(progress)
    # Syllabus
    find_invalid_links(self.course.syllabus_body) do |links|
      self.issues << {:name => I18n.t(:syllabus, "Course Syllabus"), :type => :syllabus,
                 :content_url => "/courses/#{self.course.id}/assignments/syllabus"}.merge(:invalid_links => links)
    end
    progress.update_completion! 5

    # Assessment questions
    self.course.assessment_questions.active.each do |aq|
      next if aq.assessment_question_bank.deleted?
      check_question(aq)
    end
    progress.update_completion! 15

    # Assignments
    self.course.assignments.active.each do |assignment|
      next if assignment.quiz || assignment.discussion_topic
      find_invalid_links(assignment.description) do |links|
        self.issues << {:name => assignment.title, :type => :assignment,
                   :content_url => "/courses/#{self.course.id}/assignments/#{assignment.id}"}.merge(:invalid_links => links)
      end
    end
    progress.update_completion! 25

    # Calendar events
    self.course.calendar_events.active.each do |event|
      find_invalid_links(event.description) do |links|
        self.issues << {:name => event.title, :type => :calendar_event,
                   :content_url => "/courses/#{self.course.id}/calendar_events/#{event.id}"}.merge(:invalid_links => links)
      end
    end
    progress.update_completion! 35

    # Discussion topics
    self.course.discussion_topics.active.each do |topic|
      find_invalid_links(topic.message) do |links|
        self.issues << {:name => topic.title, :type => :discussion_topic,
                   :content_url => "/courses/#{self.course.id}/discussion_topics/#{topic.id}"}.merge(:invalid_links => links)
      end
    end
    progress.update_completion! 55

    # External URL Module items (almost forgot about these)
    self.course.context_module_tags.not_deleted.where(:content_type => "ExternalUrl").each do |ct|
      find_invalid_link(ct.url) do |invalid_link|
        self.issues << {:name => ct.title, :type => :module_item,
                   :content_url => "/courses/#{self.course.id}/modules"}.merge(:invalid_links => [invalid_link])
      end
    end
    progress.update_completion! 65

    # Quizzes
    self.course.quizzes.active.each do |quiz|
      find_invalid_links(quiz.description) do |links|
        self.issues << {:name => quiz.title, :type => :quiz,
                   :content_url => "/courses/#{self.course.id}/quizzes/#{quiz.id}"}.merge(:invalid_links => links)
      end
      quiz.quiz_questions.each do |qq|
        check_question(qq)
      end
    end
    progress.update_completion! 85

    # Wiki pages
    self.course.wiki.wiki_pages.not_deleted.each do |page|
      find_invalid_links(page.body) do |links|
        self.issues << {:name => page.title, :type => :wiki_page,
                   :content_url => "/courses/#{self.course.id}/pages/#{page.url}"}.merge(:invalid_links => links)
      end
    end
    progress.update_completion! 99
  end

  def check_question(question)
    # Assessment/Quiz Questions

    links = []
    [:question_text, :correct_comments_html, :incorrect_comments_html, :neutral_comments_html, :more_comments_html].each do |field|
      find_invalid_links(question.question_data[field]) do |field_links|
        links += field_links
      end
    end

    (question.question_data[:answers] || []).each_with_index do |answer, i|
      [:html, :comments_html, :left_html].each do |field|
        find_invalid_links(answer[field]) do |field_links|
          links += field_links
        end
      end
    end

    if links.any?
      hash = {:name => question.question_data[:question_name]}.merge(:invalid_links => links)
      case question
      when AssessmentQuestion
        hash[:type] = :assessment_question
        hash[:content_url] = "/courses/#{self.course.id}/question_banks/#{question.assessment_question_bank_id}#question_#{question.id}_question_text"
      when Quizzes::QuizQuestion
        hash[:type] = :quiz_question
        hash[:content_url] = "/courses/#{self.course.id}/quizzes/#{question.quiz_id}/take?preview=1#question_#{question.id}"
      end
      issues << hash
    end
  end

  # pretty much copied from ImportedHtmlConverter
  def find_invalid_links(html)
    links = []
    doc = Nokogiri::HTML(html || "")
    attrs = ['rel', 'href', 'src', 'data', 'value']

    doc.search("*").each do |node|
      attrs.each do |attr|
        url = node[attr]
        next unless url.present?
        if attr == 'value'
          next unless node['name'] && node['name'] == 'src'
        end

        find_invalid_link(url) do |invalid_link|
          links << invalid_link
        end
      end
    end

    yield links if links.any?
  end

  ITEM_CLASSES = {
    'assignments' => Assignment,
    'announcements' => Announcement,
    'calendar_events' => CalendarEvent,
    'discussion_topics' => DiscussionTopic,
    'collaborations' => Collaboration,
    'files' => Attachment,
    'quizzes' => Quizzes::Quiz,
    'groups' => Group,
    'wiki' => WikiPage,
    'pages' => WikiPage,
    'modules' => ContextModule,
    'items' => ContentTag
  }

  # yields a hash containing the url and an error type if the url is invalid
  def find_invalid_link(url)
    unless result = self.visited_urls[url]
      begin
        if ImportedHtmlConverter.relative_url?(url) || (self.domain_regex && url.match(self.domain_regex))

          if valid_route?(url)
            if url.match(/\/courses\/(\d+)/) && self.course.id.to_s != $1
              result = :course_mismatch
            else
              result = check_object_status(url)
            end
          else
            result = :unreachable
          end
        elsif !url.start_with?('mailto:')
          unless reachable_url?(url)
            result = :unreachable
          end
        end
      rescue URI::Error
        result = :unparsable
      end
      result ||= :success
      self.visited_urls[url] = result
    end

    unless result == :success
      invalid_link = {:url => url, :reason => result}
      yield invalid_link
    end
  end

  # checks against the Rails routes to see if the url matches anything
  def valid_route?(url)
    path = URI.parse(url).path
    path = path.chomp("/")

    @route_set ||= ::Rails.application.routes.set.routes.select{|r| r.verb === "GET"}
    @route_set.any?{|r| r.path.match(path)} || (!Pathname(path).each_filename.include?('..') && File.exists?(File.join(Rails.root, "public", path)))
  end

  # makes sure that links to course objects exist and are in a visible state
  def check_object_status(url)
    result = nil
    case url
    when /\/courses\/\d+\/file_contents\/(.*)/
      rel_path = CGI.unescape($1)
      unless (att = Folder.find_attachment_in_context_with_path(self.course, rel_path)) && !att.deleted?
        result = :missing_file
      end
    when /\/courses\/\d+\/(pages|wiki)\/([^\s"<'\?\/#]*)/
      if obj = self.course.wiki.find_page(CGI.unescape($2))
        if obj.workflow_state == 'unpublished'
          result = :unpublished_item
        end
      else
        result = :missing_item
      end
    when /\/courses\/\d+\/(.*)\/(\d+)/
      obj_type =  $1
      obj_id = $2

      if obj_class = ITEM_CLASSES[obj_type]
        if (obj_class == Attachment) && (obj = self.course.attachments.find_by_id(obj_id)) # attachments.find_by_id uses the replacement hackery
          if obj.file_state == 'deleted'
            result = :missing_item
          elsif obj.locked?
            result = :unpublished_item
          end
        elsif (obj = obj_class.where(:id => obj_id).first)
          if obj.workflow_state == 'deleted'
            result = :missing_item
          elsif obj.workflow_state == 'unpublished'
            result = :unpublished_item
          end
        else
          result = :missing_item
        end
      end
    end
    result
  end

  # ping the url and make sure we get a 200
  def reachable_url?(url)
    begin
      response = CanvasHttp.head(url, { "Accept-Encoding" => "gzip" }, redirect_limit: 9)
      if %w{404 405}.include?(response.code)
        response = CanvasHttp.get(url, { "Accept-Encoding" => "gzip" }, redirect_limit: 9)
      end

      case response.code
      when /^2/ # 2xx code
        true
      when "401", "403", "503"
        # we accept unauthorized and forbidden codes here because sometimes servers refuse to serve our requests
        # and someone can link to a site that requires authentication anyway - doesn't necessarily make it invalid
        true
      else
        false
      end
    rescue
      false
    end
  end
end