{"domain": "microsoftai.com", "user": "Import microsoftai.com", "isWillingToRelocate": false, "isUserAuthenticated": false, "isUserETXCandidate": false, "isDomainETX": false, "isCareerPlannerEnabled": false, "isMyApplicationsEnabled": false, "showVeteranEmployerSignUp": false, "candidate": {"enc_id": 0, "fullname": "", "firstname": "", "lastname": "", "skills": [], "email": "", "phone": "", "location": "", "filename": null, "starred_positions": [], "resumeUrl": "", "onboardingCompleted": false, "isUserInPcsIjp": false, "linkedinUrl": ""}, "branding": {"advancedSearchOptions": [{"key": "skills", "name": "Skills"}, {"key": "work flex", "name": "Work Site"}, {"key": "profession", "name": "Profession"}, {"key": "role type", "name": "Role Type"}, {"key": "discipline", "name": "Discipline"}, {"key": "location country", "name": "Country"}], "redesignedNuxConfig": {"enabled": false}, "hideJobCart": true, "showUploadResumeModal": 0, "enableTalentNetwork": 1, "show_wizard": 0, "privacy": {"title": "Data Privacy Notice", "logged_out_notifications_text": " ", "text": "Your privacy is important to us. The personal data you provide will be processed by AI to enhance your experience. For more information, please see the <u><a class=\"msft-data-privacy-link\" href=\"http://go.microsoft.com/fwlink/?LinkId=518021\" target=\"_blank\">Microsoft Data Privacy Notice</a></u> and <u><a class=\"msft-data-privacy-link\" href=\"https://aka.ms/MAIHubTransparency\" target=\"_blank\">Transparency FAQ</a></u>.", "button": "I understand", "logged_out_notifications_privacy_policy_checkbox_default_state": true}, "homePageHeroBanner": {"opacity": 1, "image": "https://static.vscdn.net/images/careers/demo/microsoft-ai-sandbox/1731551848::pcsbanner", "title": "Get personalized job recommendations", "hideInMobileView": false}, "navBar": {"color": "#ffffff", "image": "https://static.vscdn.net/images/careers/demo/microsoft-ta2-dev-sandbox/1719948341::1714657312__msft_logo.png", "link": "http://eightfold.ai", "background": "#0346A2", "opacity": 1}, "uploadResumeModal": {"title": "Welcome to {company_name}'s Career Center", "subtitle": "Streamline your search by uploading your resume to be matched with positions that best suit your qualifications.", "disclaimer": "**Uploading a resume is not a formal application for employment**"}, "custom_style": {"css": ".nav-item a { color: #fff !important;} .refer .user-name { color: #fff !important; } .fixed-top { background-color: #000 !important; } .position-job-description .block__field--wide-rich-text { width: inherit !important; } nav.navbar { display: none !important; }\n.msft-accessiblity-link {color: #5c1b86; text-decoration:underline} .msft-data-privacy-link {color: #5c1b86; text-decoration:underline} .position-job-description a { color: #5c1b86 !important; text-decoration:underline !important }"}, "page_image": "", "max_applications_refer": 0, "max_applications_apply": 20, "custom_head_scripts": {"HOME_PAGE_LOAD": "<script>$(\"#pcs-body-container\").removeClass(\"without-hero-image\");\n        $(\"nav.navbar\").hide();\n $('.hero-image').hide();\n $('.language-selection').hide();\n$('.language-selection').css('display', 'none');</script>", "SINGLE_POSITION_PAGE_LOAD": "<script>$(\"#pcs-body-container\").removeClass(\"without-hero-image\");\n        $(\"nav.navbar\").hide();\n $('.hero-image').hide();\n $('.language-selection').hide();\n$('.language-selection').css('display', 'none');</script>"}, "applyButton": {"background": "#5c1b86"}, "customContent": {"positionSections": [{"body": "Microsoft is an equal opportunity employer. All qualified applicants will receive consideration for employment without regard to age, ancestry, citizenship, color, family or medical care leave, gender identity or expression, genetic information, immigration status, marital status, medical condition, national origin, physical or mental disability, political affiliation, protected veteran or military status, race, ethnicity, religion, sex (including pregnancy), sexual orientation, or any other characteristic protected by applicable local laws, regulations and ordinances. If you need assistance and/or a reasonable accommodation due to a disability during the application process, read more about <a class=\"msft-accessiblity-link\" href=\"https://careers.microsoft.com/v2/global/en/accessibility.html\" target=\"_blank\">requesting accommodations</a>.", "title": ""}]}, "job_insights_display": {"hide": true}, "hide_matched_section": true, "PersonalizationBar": {"hide_find_jobs_button": true}, "custom_html": {"header": "<div id=\"configurable-navbar\"></div>", "footer": "<footer></footer>", "page-footer": "<div></div>"}, "recaptcha_enabled": 1, "companyName": "Microsoft", "showLoggedOutNotificationsPrivacyPolicy": true, "hideEightfoldBranding": false}, "pid": "1970324837016564", "positions": [{"id": 1970324837016564, "name": "Senior Product Manager, Copilot AI", "location": "Mountain View, California, United States", "locations": ["Mountain View, California, United States", "Redmond, WA, USA"], "hot": 0, "department": "AI Research Team (9M6I)", "business_unit": "Microsoft AI (QIA5)", "t_update": 1749089179, "t_create": 1748368272, "ats_job_id": 1970324837016564, "display_job_id": 1970324837016564, "type": "ATS", "id_locale": "1970324837016564-en", "job_description": "Define the end-to-end evaluation strategy for LLM-based product, spanning defining metrics, building datasets, synthesize insights and translate them into actionable steps for product response quality improvement. Design and run human evaluations, write evaluation guidelines, and ensure evaluation quality. Design and write LLM-as-judge prompts, build and run machine evaluations. Collaborate with engineers, and product teams to drive eval implementations and share best practices across the board. Partner with data and insights teams to develop scalable pipelines and dashboards to track pre-launch eval performance across all dimensions. Own product requirements and roadmaps for internal tools used to run and analyze evaluations at scale. Translate evaluation insights into actionable product improvements and inform product quality improvement decisions. Monitor trends in product performance, user feedback, and competitive benchmarks to continuously refine evaluation approaches. Own key projects, proactively identifying risks and proposing solutions to ensure timely delivery. Coordinate Cross-Team Collaboration: Work closely with engineering, data science, and product stakeholders to align on goals, track deliverables, and communicate status updates effectively. Embody our Culture and Values. Bachelor's Degree AND 5+ years experience in product/technical program management or software development OR equivalent experience. 3+ years of experience leading multi-disciplinary projects, defining requirements, developing project plans and working with multi-disciplinary teams to execute them. Experience in building and evaluating ML-powered or LLM-powered products. Computer science background, experience in training/ evaluation of LLMs Technical depth in software development, data science and machine learning. While you are not expected to write code on the critical path, you're able to define and execute on zero-to-one prototypes, reason over large amounts of data, and speak the language of the engineers and researchers you work with. Experience using and applying Large Language Models. LLMs will be your clay - you should know how to prompt engineer them, how to evaluate them, and understand how they are tuned.", "stars": 0.0, "medallionProgram": null, "location_flexibility": null, "work_location_option": "onsite", "canonicalPositionUrl": "https://microsoftai.eightfold.ai/careers/job/1970324837016564", "isPrivate": false}], "isFallback": false, "debug": {}, "count": 1, "personal_message": "We thought you would be interested in this position.", "scheduling": {"minTime": 9, "limit": 10, "maxTime": 17, "increments": 60, "minTimeslots": 3}, "userTitles": [], "enableTargetedResume": 0, "query": {"query": "", "location": "", "department": [], "skill": [], "seniority": [], "pid": "1970324837016564"}, "singleview": true, "see_all_jobs": true, "recommended_star_threshold": 4.0, "chatbot": false, "iframeImplementation": null, "pcsApplyFormV2Enabled": true, "isPcsBrandingApril2023Enabled": false, "allowedFileTypes": {}, "pcsTextConfiguration": {}, "hideDepartment": null, "pcsOctupleMigration0Enabled": true, "pcsOctupleMigration1Enabled": false, "replaceUrlOnGoBack": true, "pcsRedesignedNuxEnabled": false, "readmoreInstructionEnabled": false, "userActivityTimeout": 86400000, "userActivityTimeoutEnabled": 1, "isLoggedInPcsEnabled": false, "sortByConfig": null, "searchBoxConfig": {}, "excludePrivatePositions": true, "eeocFilterKeywords": ["veteran", "disability", "gender", "race", "citizen", "visa", "ethnicity"], "disableScrollLoadPositionSidebar": false, "locationFlexibilityFrontendEnabled": false, "workLocationOptionFrontendEnabled": true, "remoteFlexibleJobsFilterEnabled": false, "loggedOutNotificationsEnabled": true, "candidateLogin": {}, "prepopulateApplyFormEnabled": true, "prepopulateSettings": {"inferEEOQuestions": false, "showPrepopulateCheckbox": true, "defaultPrepopulateCheckboxValue": true, "showClearForm": true, "prepopulateCheckboxText": "Save my answers for future applications"}, "candidateBuildProfile": {}, "enhancementsEnabled": false, "themeBuilderUser": null, "mandatoryFields": ["firstname", "lastname", "email", "phone"], "blindfoldWidgetPcsGate": false, "pcsApplyFormLocationGate": false, "t3sEnabled": true, "uploadApplicationAnswers": false, "candidateAuthV2Enabled": true, "preApplicationSubmitAuthEnabled": false, "applyFormV2Enabled": false, "loggedOutSavedSearchEnabled": false, "locationRadiusTypeToggleEnabled": true, "incompleteApplicationsEnabled": false, "incompleteApplicationConfig": {}, "fallbackPcsJdGate": true, "enableResumeCoach": false, "isPcsEnabled": true, "applicationInfoReviewEnabled": false, "phoneWithCountryCodeEnabled": true, "phoneWithCountryCodeJTNEnabled": false, "notificationSuggestVerificationToken": null, "cookiesAutoDisabled": false, "strictEmailValidationEnabled": true, "chatbotxConfig": {"enabled": false, "featureAccessFlags": {"resumeCoachCardFlags": {"showApplyWithResume": true, "showEditResume": true}, "positionCardFlags": {"showAddToJobCart": false}}}, "pcsOptionalResumeWithJobcartGate": false, "loggedInCandidate": {}, "hamburgerMenuEnabled": false, "sharedTalentPoolGate": false, "pcsAccessibilityHomeEnabled": true, "pcsAccessibilityApplyFormEnabled": true, "showLanguageDropdown": false, "languages": [{"value": "en", "title": "English"}, {"value": "it", "title": "Italiano"}, {"value": "es", "title": "Espa\u00f1ol"}, {"value": "fr", "title": "Fran\u00e7ais"}, {"value": "pt", "title": "Portugu\u00eas"}, {"value": "nb", "title": "Norsk"}, {"value": "de", "title": "Deutsch"}, {"value": "ja", "title": "\u65e5\u672c\u8a9e"}, {"value": "ms", "title": "Bahasa melayu"}, {"value": "zh-CN", "title": "\u4e2d\u6587 (\u7b80\u4f53)"}, {"value": "zh-TW", "title": "\u4e2d\u6587 (\u7e41\u9ad4)"}, {"value": "ko", "title": "\ud55c\uad6d\uc5b4"}, {"value": "th", "title": "\u0e20\u0e32\u0e29\u0e32\u0e44\u0e17\u0e22"}, {"value": "nl", "title": "Nederlands"}, {"value": "pl", "title": "Polski"}, {"value": "uk", "title": "Y\u043a\u0440\u0430\u0457\u043d\u0441\u044c\u043a\u0430"}, {"value": "hr", "title": "Hrvatski"}, {"value": "el", "title": "\u0395\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac"}, {"value": "hu", "title": "Magyar"}, {"value": "cs", "title": "\u010de\u0161tina"}, {"value": "tr", "title": "T\u00fcrk\u00e7e"}, {"value": "ru", "title": "P\u0443\u0441\u0441\u043a\u0438\u0439"}, {"value": "ht", "title": "Haitian"}, {"value": "he", "title": "\u05e2\u05d1\u05e8\u05d9\u05ea"}, {"value": "pt-BR", "title": "Brazilian Portugu\u00eas"}, {"value": "da", "title": "Dansk"}, {"value": "fi", "title": "Suomi"}, {"value": "sv", "title": "Svenska"}, {"value": "fr-CA", "title": "Fran\u00e7ais (Canada)"}, {"value": "pt-PT", "title": "portugu\u00eas (Portugal)"}, {"value": "vi", "title": "Ti\u1ebfng Vi\u1ec7t"}, {"value": "ro", "title": "rom\u00e2n\u0103"}, {"value": "bg", "title": "\u0431\u044a\u043b\u0433\u0430\u0440\u0441\u043a\u0438"}, {"value": "ar", "title": "\u0627\u0644\u0639\u0631\u0628\u064a\u0629"}, {"value": "es-419", "title": "Espa\u00f1ol - Am\u00e9rica Latina"}], "displayLanguage": "en", "installed_app_data": [], "singlePageCareersNavbarGate": false, "customJobDescTranslationSkipList": [], "all_applicable_locations": [{"location": "Mountain View, California, United States", "city": "Mountain View", "state": "CA,US", "country": "US"}, {"location": "Redmond, WA, USA", "city": "Redmond", "state": "WA,US", "country": "US"}]}